From 2b763294ae925438b3447e674dd144c5c76b0d48 Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Wed, 19 Aug 2026 10:08:41 +0800 Subject: [PATCH 1/7] docs: add Feast offline-to-online notebook --- .../feast-offline-to-online-inference.ipynb | 530 ++++++++++++++++++ docs/en/train/guides/index.mdx | 1 + e2e/cases/c16_feast_offline_online.sh | 251 +++++++++ e2e/lib.sh | 9 +- e2e/run_all.sh | 3 + 5 files changed, 793 insertions(+), 1 deletion(-) create mode 100644 docs/en/train/guides/feast-offline-to-online-inference.ipynb create mode 100755 e2e/cases/c16_feast_offline_online.sh diff --git a/docs/en/train/guides/feast-offline-to-online-inference.ipynb b/docs/en/train/guides/feast-offline-to-online-inference.ipynb new file mode 100644 index 00000000..68082f9f --- /dev/null +++ b/docs/en/train/guides/feast-offline-to-online-inference.ipynb @@ -0,0 +1,530 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Feast offline-to-online training and inference\n", + "\n", + "This notebook follows the [Feast quickstart](../../develop/components/feast/quickstart.mdx) and runs a small, CPU-only demonstration:\n", + "\n", + "1. create a Parquet batch dataset;\n", + "2. define and register Feast entities, feature views, and a feature service;\n", + "3. retrieve historical features and train a sample NumPy model;\n", + "4. materialize the same features into the Feast online store; and\n", + "5. deploy a KServe `InferenceService` whose model server reads the online features before predicting.\n", + "\n", + "The notebook assumes `kubectl`, Feast 0.61.x, NumPy, pandas, PyArrow, `requests`, and KServe are available. It uses a PVC for the model artifact and the published Feast feature-server image for the custom serving runtime. The same flow is automated by [`e2e/cases/c16_feast_offline_online.sh`](../../../e2e/cases/c16_feast_offline_online.sh)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import base64\n", + "import json\n", + "import os\n", + "import subprocess\n", + "import textwrap\n", + "import time\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import requests\n", + "\n", + "NAMESPACE = os.environ.get(\"FEAST_NAMESPACE\", \"feast-demo\")\n", + "FEATURESTORE_NAME = os.environ.get(\"FEAST_FEATURESTORE\", \"feast-notebook\")\n", + "FEAST_PROJECT = os.environ.get(\"FEAST_PROJECT\", \"feast_demo\")\n", + "MODEL_PVC = os.environ.get(\"FEAST_MODEL_PVC\", \"feast-notebook-model\")\n", + "MODEL_RUNTIME = os.environ.get(\"FEAST_MODEL_RUNTIME\", \"feast-numpy-runtime\")\n", + "MODEL_NAME = os.environ.get(\"FEAST_MODEL_NAME\", \"feast-online-model\")\n", + "MODEL_IMAGE = os.environ.get(\"FEAST_MODEL_IMAGE\", \"build-harbor.alauda.cn/mlops/feast/feature-server:0.61.0\")\n", + "REPO = Path(\"feast-notebook-repo\")\n", + "DATA_DIR = REPO / \"data\"\n", + "MODEL_DIR = Path(\"feast-notebook-model\")\n", + "REPO.mkdir(exist_ok=True)\n", + "DATA_DIR.mkdir(parents=True, exist_ok=True)\n", + "MODEL_DIR.mkdir(exist_ok=True)\n", + "\n", + "def kubectl(*args, input_text=None, check=True):\n", + " result = subprocess.run([\"kubectl\", *args], input=input_text, text=True, capture_output=True)\n", + " if check and result.returncode:\n", + " raise RuntimeError(f\"kubectl {' '.join(args)} failed: {result.stderr}\")\n", + " return result.stdout.strip()\n", + "\n", + "print({\"namespace\": NAMESPACE, \"featurestore\": FEATURESTORE_NAME, \"model_pvc\": MODEL_PVC})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Prepare the FeatureStore operand\n", + "\n", + "The quickstart requires `registry.local.server: {}` when a client outside the Feast pod runs `feast apply`. The UI is optional for this workflow, but is enabled here for inspection. If you already have a suitable `FeatureStore`, set `FEAST_FEATURESTORE` and skip this cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for namespace in (NAMESPACE, \"feast-operator-system\"):\n", + " namespace_yaml = kubectl(\"create\", \"namespace\", namespace, \"--dry-run=client\", \"-o\", \"yaml\")\n", + " kubectl(\"apply\", \"-f\", \"-\", input_text=namespace_yaml)\n", + "featurestore_yaml = f\"\"\"\n", + "apiVersion: feast.dev/v1\n", + "kind: FeatureStore\n", + "metadata:\n", + " name: {FEATURESTORE_NAME}\n", + " namespace: {NAMESPACE}\n", + "spec:\n", + " feastProject: {FEAST_PROJECT}\n", + " services:\n", + " registry:\n", + " local:\n", + " server: {{}}\n", + " ui: {{}}\n", + "\"\"\"\n", + "kubectl(\"apply\", \"-f\", \"-\", input_text=featurestore_yaml)\n", + "deadline = time.time() + 600\n", + "while time.time() < deadline:\n", + " phase = kubectl(\"get\", \"featurestore\", FEATURESTORE_NAME, \"-n\", NAMESPACE, \"-o\", \"jsonpath={.status.phase}\", check=False)\n", + " print(phase or \"Pending\")\n", + " if phase == \"Ready\":\n", + " break\n", + " if phase == \"Failed\":\n", + " raise RuntimeError(kubectl(\"describe\", \"featurestore\", FEATURESTORE_NAME, \"-n\", NAMESPACE, check=False))\n", + " time.sleep(10)\n", + "else:\n", + " raise TimeoutError(\"FeatureStore did not become Ready\")\n", + "\n", + "client_config_map = kubectl(\"get\", \"featurestore\", FEATURESTORE_NAME, \"-n\", NAMESPACE, \"-o\", \"jsonpath={.status.clientConfigMap}\")\n", + "client_config = kubectl(\"get\", \"configmap\", client_config_map, \"-n\", NAMESPACE, \"-o\", r\"jsonpath={.data.feature_store\\.yaml}\")\n", + "print(\"FeatureStore is Ready; client config length:\", len(client_config))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Prepare a Parquet batch source\n", + "\n", + "The event timestamp is required by Feast for point-in-time historical retrieval. The label is kept in the entity dataframe for model training and is not registered as a feature." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rng = np.random.default_rng(7)\n", + "n_rows = 240\n", + "events = pd.DataFrame({\n", + " \"driver_id\": (np.arange(n_rows) % 12 + 1).astype(\"int64\"),\n", + " \"event_timestamp\": pd.date_range(\"2026-01-01\", periods=n_rows, freq=\"h\", tz=\"UTC\"),\n", + "})\n", + "events[\"created\"] = events[\"event_timestamp\"] + pd.to_timedelta(1, unit=\"m\")\n", + "events[\"conv_rate\"] = (0.25 + 0.55 * rng.random(n_rows)).astype(\"float32\")\n", + "events[\"acc_rate\"] = (0.50 + 0.45 * rng.random(n_rows)).astype(\"float32\")\n", + "events[\"avg_daily_trips\"] = rng.integers(2, 20, size=n_rows).astype(\"int64\")\n", + "events[\"label\"] = ((events[\"conv_rate\"] * 2 + events[\"acc_rate\"] + events[\"avg_daily_trips\"] / 20) > 1.8).astype(\"int64\")\n", + "parquet_path = DATA_DIR / \"driver_stats.parquet\"\n", + "events.to_parquet(parquet_path, index=False)\n", + "print(parquet_path, events.shape)\n", + "events.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Define and register offline features\n", + "\n", + "This follows the quickstart’s `Entity` + `FileSource` + `FeatureView` + `FeatureService` pattern. For the operand’s default PVC-backed SQLite online store, the cell also runs `feast apply` inside the online-store pod once, creating the feature-view table before remote materialization writes to it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "(REPO / \"features.py\").write_text(textwrap.dedent(\"\"\"\n", + " from datetime import timedelta\n", + " from feast import Entity, FeatureService, FeatureView, Field, FileSource\n", + " from feast.data_format import ParquetFormat\n", + " from feast.types import Float32, Int64\n", + " from feast.value_type import ValueType\n", + "\n", + " driver = Entity(name=\"driver\", join_keys=[\"driver_id\"], value_type=ValueType.INT64)\n", + " driver_stats_source = FileSource(\n", + " name=\"driver_stats_source\", path=\"data/driver_stats.parquet\",\n", + " file_format=ParquetFormat(), timestamp_field=\"event_timestamp\",\n", + " created_timestamp_column=\"created\",\n", + " )\n", + " driver_hourly_stats = FeatureView(\n", + " name=\"driver_hourly_stats\", entities=[driver], ttl=timedelta(days=365),\n", + " schema=[\n", + " Field(name=\"conv_rate\", dtype=Float32),\n", + " Field(name=\"acc_rate\", dtype=Float32),\n", + " Field(name=\"avg_daily_trips\", dtype=Int64),\n", + " ], online=True, source=driver_stats_source,\n", + " )\n", + " driver_activity_v1 = FeatureService(name=\"driver_activity_v1\", features=[driver_hourly_stats])\n", + "\"\"\"))\n", + "\n", + "# Copy the platform-generated config and make the online/registry certificates\n", + "# available to this notebook process. The serving pod will mount the original\n", + "# /tls paths; keep that copy separately for the model artifact.\n", + "runtime_config = client_config\n", + "local_config = runtime_config\n", + "for secret_name, original_path, local_name in [\n", + " (f\"feast-{FEATURESTORE_NAME}-online-tls\", \"/tls/online/tls.crt\", \"online-tls.crt\"),\n", + " (f\"feast-{FEATURESTORE_NAME}-registry-tls\", \"/tls/registry/tls.crt\", \"registry-tls.crt\"),\n", + "]:\n", + " cert_b64 = kubectl(\"get\", \"secret\", secret_name, \"-n\", NAMESPACE, \"-o\", r\"jsonpath={.data.tls\\.crt}\", check=False)\n", + " if cert_b64:\n", + " cert_path = (REPO / local_name).resolve()\n", + " cert_path.write_bytes(base64.b64decode(cert_b64))\n", + " local_config = local_config.replace(original_path, str(cert_path))\n", + "(REPO / \"feature_store.yaml\").write_text(local_config)\n", + "(MODEL_DIR / \"feature_store.yaml\").write_text(runtime_config)\n", + "(MODEL_DIR / \"features.py\").write_text((REPO / \"features.py\").read_text())\n", + "subprocess.run([\"feast\", \"--chdir\", str(REPO), \"apply\"], check=True)\n", + "\n", + "# Feast 0.61's remote online-store client does not create SQLite tables.\n", + "# Apply once in the operand's local repository to create that infrastructure.\n", + "online_pod = kubectl(\"get\", \"pods\", \"-n\", NAMESPACE, \"-l\", f\"feast.dev/name={FEATURESTORE_NAME}\",\n", + " \"-o\", \"jsonpath={.items[0].metadata.name}\")\n", + "if not online_pod:\n", + " raise RuntimeError(\"FeatureStore online pod was not found\")\n", + "kubectl(\"cp\", str(REPO / \"features.py\"),\n", + " f\"{NAMESPACE}/{online_pod}:/feast-data/{FEAST_PROJECT}/feature_repo/feature_definitions.py\",\n", + " \"-c\", \"online\")\n", + "kubectl(\"exec\", \"-n\", NAMESPACE, online_pod, \"-c\", \"online\", \"--\", \"bash\", \"-c\",\n", + " f\"cd /feast-data/{FEAST_PROJECT}/feature_repo && feast apply\")\n", + "print(\"Feature definitions registered; online-store infrastructure created\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Retrieve offline features and train a sample model\n", + "\n", + "The model is deliberately a small NumPy linear classifier so the example does not require a second private training image. The training matrix comes from Feast’s historical feature retrieval, not directly from the raw Parquet columns." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from feast import FeatureStore\n", + "\n", + "store = FeatureStore(repo_path=str(REPO))\n", + "entity_df = events[[\"driver_id\", \"event_timestamp\", \"label\"]].copy()\n", + "training_df = store.get_historical_features(\n", + " entity_df=entity_df,\n", + " features=[\n", + " \"driver_hourly_stats:conv_rate\",\n", + " \"driver_hourly_stats:acc_rate\",\n", + " \"driver_hourly_stats:avg_daily_trips\",\n", + " ],\n", + ").to_df().dropna()\n", + "\n", + "feature_columns = [\"conv_rate\", \"acc_rate\", \"avg_daily_trips\"]\n", + "X = training_df[feature_columns].to_numpy(dtype=\"float64\")\n", + "y = training_df[\"label\"].to_numpy(dtype=\"float64\")\n", + "X_bias = np.column_stack([np.ones(len(X)), X])\n", + "weights = np.linalg.pinv(X_bias) @ y\n", + "np.savez(MODEL_DIR / \"model.npz\", weights=weights, feature_columns=np.array(feature_columns))\n", + "print(\"historical rows:\", len(training_df), \"weights:\", weights)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Materialize and verify online features\n", + "\n", + "`materialize_incremental` copies the registered batch features into the online store. The inference server below uses the same feature service and entity key at request time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "end_date = events[\"event_timestamp\"].max().to_pydatetime() + pd.Timedelta(hours=1)\n", + "store.materialize_incremental(end_date)\n", + "online = store.get_online_features(\n", + " features=store.get_feature_service(\"driver_activity_v1\"),\n", + " entity_rows=[{\"driver_id\": 1}, {\"driver_id\": 2}],\n", + ").to_df()\n", + "online" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Define the online model server\n", + "\n", + "The server loads the trained NumPy weights from the model PVC, queries Feast online features, and returns a KServe v2 response. The Feast online and registry TLS secrets are mounted by the `ServingRuntime`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_source = textwrap.dedent(\"\"\"\n", + " import os\n", + " import numpy as np\n", + " from fastapi import Body, FastAPI\n", + " from feast import FeatureStore\n", + " import uvicorn\n", + "\n", + " MODEL_NAME = os.getenv(\"MODEL_NAME\", \"feast-online-model\")\n", + " weights = np.load(\"/mnt/models/model.npz\")[\"weights\"]\n", + " store = FeatureStore(repo_path=\"/mnt/models\")\n", + " feature_service = store.get_feature_service(\"driver_activity_v1\")\n", + " app = FastAPI()\n", + "\n", + " @app.get(\"/v2/health/live\")\n", + " @app.get(\"/v2/health/ready\")\n", + " def ready():\n", + " return {\"ready\": True}\n", + "\n", + " @app.get(\"/v2/models/{model_name}\")\n", + " @app.get(\"/v2/models/{model_name}/ready\")\n", + " def model_ready(model_name: str):\n", + " return {\"name\": model_name, \"ready\": model_name == MODEL_NAME}\n", + "\n", + " @app.post(\"/v2/models/{model_name}/infer\")\n", + " def infer(model_name: str, payload: dict = Body(...)):\n", + " ids = next(item for item in payload[\"inputs\"] if item[\"name\"] == \"driver_id\")[\"data\"]\n", + " rows = [{\"driver_id\": int(driver_id)} for driver_id in ids]\n", + " values = store.get_online_features(features=feature_service, entity_rows=rows).to_dict()\n", + " def column(name):\n", + " if name in values:\n", + " return values[name]\n", + " return values[next(key for key in values if key.endswith(\"__\" + name))]\n", + " X = np.column_stack([np.ones(len(ids)), column(\"conv_rate\"), column(\"acc_rate\"), column(\"avg_daily_trips\")])\n", + " prediction = (X @ weights).astype(\"float32\")\n", + " return {\"model_name\": model_name, \"outputs\": [{\"name\": \"prediction\", \"shape\": [len(ids)], \"datatype\": \"FP32\", \"data\": prediction.tolist()}]}\n", + "\n", + " if __name__ == \"__main__\":\n", + " uvicorn.run(app, host=\"0.0.0.0\", port=8080)\n", + "\"\"\")\n", + "(MODEL_DIR / \"server.py\").write_text(server_source)\n", + "print(MODEL_DIR / \"server.py\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Stage the model and start KServe\n", + "\n", + "Create a model PVC before running this section. The temporary stager pod copies the local model artifact into it; KServe then consumes it through `storageUri: pvc://...`. The runtime mounts the Feast online and registry certificates at the paths referenced by the generated client configuration. The `RawDeployment` annotation allows a direct predictor Service where the cluster permits it; the final cell verifies the predictor Deployment itself so it also works on clusters whose KServe policy selects Standard mode." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pvc_yaml = f\"\"\"\n", + "apiVersion: v1\n", + "kind: PersistentVolumeClaim\n", + "metadata:\n", + " name: {MODEL_PVC}\n", + " namespace: {NAMESPACE}\n", + "spec:\n", + " accessModes: [ReadWriteOnce]\n", + " resources:\n", + " requests:\n", + " storage: 1Gi\n", + "\"\"\"\n", + "kubectl(\"apply\", \"-f\", \"-\", input_text=pvc_yaml)\n", + "stager = f\"\"\"\n", + "apiVersion: v1\n", + "kind: Pod\n", + "metadata:\n", + " name: feast-model-stager\n", + " namespace: {NAMESPACE}\n", + "spec:\n", + " restartPolicy: Never\n", + " containers:\n", + " - name: stager\n", + " image: {MODEL_IMAGE}\n", + " command: [bash, -c, sleep 3600]\n", + " volumeMounts:\n", + " - name: model\n", + " mountPath: /mnt/models\n", + " volumes:\n", + " - name: model\n", + " persistentVolumeClaim:\n", + " claimName: {MODEL_PVC}\n", + "\"\"\"\n", + "kubectl(\"apply\", \"-f\", \"-\", input_text=stager)\n", + "kubectl(\"wait\", \"--for=condition=Ready\", \"pod/feast-model-stager\", \"-n\", NAMESPACE, \"--timeout=180s\")\n", + "kubectl(\"cp\", str(MODEL_DIR / \"model.npz\"), f\"{NAMESPACE}/feast-model-stager:/mnt/models/model.npz\")\n", + "kubectl(\"cp\", str(MODEL_DIR / \"server.py\"), f\"{NAMESPACE}/feast-model-stager:/mnt/models/server.py\")\n", + "kubectl(\"cp\", str(MODEL_DIR / \"features.py\"), f\"{NAMESPACE}/feast-model-stager:/mnt/models/features.py\")\n", + "kubectl(\"cp\", str(MODEL_DIR / \"feature_store.yaml\"), f\"{NAMESPACE}/feast-model-stager:/mnt/models/feature_store.yaml\")\n", + "kubectl(\"delete\", \"pod\", \"feast-model-stager\", \"-n\", NAMESPACE, \"--wait=true\")\n", + "\n", + "runtime_yaml = f\"\"\"\n", + "apiVersion: serving.kserve.io/v1alpha1\n", + "kind: ServingRuntime\n", + "metadata:\n", + " name: {MODEL_RUNTIME}\n", + " namespace: {NAMESPACE}\n", + "spec:\n", + " containers:\n", + " - name: kserve-container\n", + " image: {MODEL_IMAGE}\n", + " command: [python, /mnt/models/server.py]\n", + " ports:\n", + " - containerPort: 8080\n", + " name: http1\n", + " protocol: TCP\n", + " env:\n", + " - name: MODEL_NAME\n", + " value: {MODEL_NAME}\n", + " volumeMounts:\n", + " - name: online-tls\n", + " mountPath: /tls/online\n", + " readOnly: true\n", + " - name: registry-tls\n", + " mountPath: /tls/registry\n", + " readOnly: true\n", + " protocolVersions: [v2]\n", + " supportedModelFormats:\n", + " - name: feast-numpy\n", + " version: \"1\"\n", + " volumes:\n", + " - name: online-tls\n", + " secret:\n", + " secretName: feast-{FEATURESTORE_NAME}-online-tls\n", + " - name: registry-tls\n", + " secret:\n", + " secretName: feast-{FEATURESTORE_NAME}-registry-tls\n", + "\"\"\"\n", + "isvc_yaml = f\"\"\"\n", + "apiVersion: serving.kserve.io/v1beta1\n", + "kind: InferenceService\n", + "metadata:\n", + " name: {MODEL_NAME}\n", + " namespace: {NAMESPACE}\n", + " annotations:\n", + " serving.kserve.io/deploymentMode: RawDeployment\n", + "spec:\n", + " predictor:\n", + " model:\n", + " modelFormat:\n", + " name: feast-numpy\n", + " version: \"1\"\n", + " protocolVersion: v2\n", + " runtime: {MODEL_RUNTIME}\n", + " storageUri: pvc://{MODEL_PVC}\n", + " resources:\n", + " requests:\n", + " cpu: \"100m\"\n", + " memory: 256Mi\n", + " limits:\n", + " cpu: \"1\"\n", + " memory: 1Gi\n", + "\"\"\"\n", + "kubectl(\"apply\", \"-f\", \"-\", input_text=runtime_yaml)\n", + "kubectl(\"apply\", \"-f\", \"-\", input_text=isvc_yaml)\n", + "print(kubectl(\"get\", \"inferenceservice\", MODEL_NAME, \"-n\", NAMESPACE))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 8. Wait for the service and send an online-feature prediction\n", + "\n", + "When the predictor Deployment has an available replica, send entity IDs to the KServe v2 endpoint. The server looks those IDs up in Feast’s online store and combines the returned features with the trained weights." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "deadline = time.time() + 900\n", + "predictor_deployment = f\"{MODEL_NAME}-predictor\"\n", + "while time.time() < deadline:\n", + " deployment = json.loads(kubectl(\"get\", \"deployment\", predictor_deployment, \"-n\", NAMESPACE, \"-o\", \"json\"))\n", + " available = deployment.get(\"status\", {}).get(\"availableReplicas\", 0) or 0\n", + " print({\"availableReplicas\": available})\n", + " if available >= 1:\n", + " break\n", + " time.sleep(10)\n", + "else:\n", + " raise TimeoutError(\"KServe predictor deployment did not become available\")\n", + "\n", + "isvc_status = json.loads(kubectl(\"get\", \"inferenceservice\", MODEL_NAME, \"-n\", NAMESPACE, \"-o\", \"json\"))\n", + "print(\"Ingress URL (if configured):\", isvc_status.get(\"status\", {}).get(\"url\"))\n", + "\n", + "port_forward = subprocess.Popen(\n", + " [\"kubectl\", \"port-forward\", f\"service/{predictor_deployment}\", \"18080:80\", \"-n\", NAMESPACE],\n", + " stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,\n", + ")\n", + "base_url = \"http://127.0.0.1:18080\"\n", + "try:\n", + " deadline = time.time() + 60\n", + " while time.time() < deadline:\n", + " if port_forward.poll() is not None:\n", + " raise RuntimeError(\"kubectl port-forward exited unexpectedly\")\n", + " try:\n", + " if requests.get(f\"{base_url}/v2/health/ready\", timeout=2).ok:\n", + " break\n", + " except requests.RequestException:\n", + " pass\n", + " time.sleep(2)\n", + " else:\n", + " raise TimeoutError(\"KServe predictor endpoint did not become ready\")\n", + "\n", + " response = requests.post(\n", + " f\"{base_url}/v2/models/{MODEL_NAME}/infer\",\n", + " json={\"inputs\": [{\"name\": \"driver_id\", \"shape\": [2], \"datatype\": \"INT64\", \"data\": [1, 2]}]},\n", + " timeout=30,\n", + " )\n", + " response.raise_for_status()\n", + " prediction = response.json()\n", + " print(json.dumps(prediction, indent=2))\n", + "finally:\n", + " port_forward.terminate()\n", + " try:\n", + " port_forward.wait(timeout=5)\n", + " except subprocess.TimeoutExpired:\n", + " port_forward.kill()\n", + " port_forward.wait()" + ] + } + ], + "metadata": { + "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, + "language_info": {"name": "python", "version": "3.11"} + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/en/train/guides/index.mdx b/docs/en/train/guides/index.mdx index af14cab8..0ff1abd7 100644 --- a/docs/en/train/guides/index.mdx +++ b/docs/en/train/guides/index.mdx @@ -22,6 +22,7 @@ End-to-end recipes for training and fine-tuning models on Alauda AI. | Interactive exploration, custom scripts, VolcanoJob submission | Workbench Notebook | [Fine-tuning LLMs using Workbench](./fine-tuning-using-notebooks.mdx) | | Full-parameter SFT / pretraining on Ascend NPU | Workbench `PyTorch CANN` / `MindSpore CANN` | [Fine-tune and Pretrain on Ascend NPU](./fine-tune-and-pretrain-llms-on-ascend-npu.mdx) | | Track a KFP run's parameters and metrics in MLflow | KFP component + MLflow SDK | [Kubeflow Pipeline + MLflow Integration](../../develop/experiment_tracking/pipelines-mlflow-integration.mdx) | +| Build features offline, materialize them online, and serve a model with Feast | Feast Operator + KServe | [Feast Offline-to-Online Inference](https://github.com/alauda/aml-docs/tree/master/docs/en/train/guides/feast-offline-to-online-inference.ipynb) | | Train tabular or time-series models with reusable AutoGluon assets | Managed KFP pipelines or composable components | [Use Reusable Kubeflow Pipeline Components](../../develop/pipelines/reusable-pipeline-components.mdx) | | Use KFP caching, parallel loops, and persistent typed artifacts | KFP 2.16.1 execution mechanisms | [Kubeflow Pipelines Execution and Storage Behavior](../../develop/pipelines/kfp-execution-and-storage.mdx) | | Daily fine-tune → evaluate → compare loop with MLflow + TrustyAI | KFP Recurring Run + MLflow Model Registry + `LMEvalJob` | [Daily Fine-Tuning Pipeline with MLflow and TrustyAI](./fine-tuning-pipeline-with-mlflow-trustyai.mdx) | diff --git a/e2e/cases/c16_feast_offline_online.sh b/e2e/cases/c16_feast_offline_online.sh new file mode 100755 index 00000000..182a7c11 --- /dev/null +++ b/e2e/cases/c16_feast_offline_online.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash +# C16: Feast Parquet -> historical features -> NumPy model -> online KServe prediction. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +source "$HERE/../lib.sh" +require_env FEAST_NAMESPACE "namespace for Feast e2e resources" +NS="$FEAST_NAMESPACE" +PROJECT="feast_demo" +RUN_ID="$(printf '%05x' $$)-$(date -u +%s)" +FS_NAME="feast-e2e-$RUN_ID" +MODEL_PVC="$FS_NAME-model" +RUNTIME="$FS_NAME-runtime" +ISVC="$FS_NAME-isvc" +JOB="$FS_NAME-job" +CM="$FS_NAME-runner" +IMAGE="$FEAST_IMAGE"; [ -n "$IMAGE" ] || IMAGE=build-harbor.alauda.cn/mlops/feast/feature-server:0.61.0 +TMP="$(mktemp -d)" +PF="" +cleanup() { + [ -n "$PF" ] && kill "$PF" 2>/dev/null || true + if [ "$FEAST_KEEP_RESOURCES" != 1 ]; then + for item in "inferenceservice $ISVC" "servingruntime $RUNTIME" "job $JOB" "configmap $CM" "pvc $MODEL_PVC" "featurestore $FS_NAME"; do + set -- $item + feast_kc -n "$NS" delete "$1" "$2" --ignore-not-found --wait=false >/dev/null 2>&1 || true + done + fi + rm -rf "$TMP" +} +trap cleanup EXIT +feast_kc get crd featurestores.feast.dev >/dev/null 2>&1 || { log "Feast CRD missing; skipping"; exit "$E2E_SKIP_RC"; } +feast_kc get crd inferenceservices.serving.kserve.io >/dev/null 2>&1 || { log "KServe CRD missing; skipping"; exit "$E2E_SKIP_RC"; } +feast_kc create namespace "$NS" --dry-run=client -o yaml | feast_kc apply -f - >/dev/null +feast_kc create namespace feast-operator-system --dry-run=client -o yaml | feast_kc apply -f - >/dev/null +cat </dev/null || true)" + [ "$phase" = Ready ] && break + [ "$phase" = Failed ] && { feast_kc -n "$NS" get featurestore "$FS_NAME" -o yaml >&2; exit 1; } + sleep 10 +done +[ "$phase" = Ready ] || { log "FeatureStore did not become Ready"; exit 1; } +CLIENT="$(feast_kc -n "$NS" get featurestore "$FS_NAME" -o jsonpath='{.status.clientConfigMap}')" +ONLINE_TLS="feast-$FS_NAME-online-tls" +REGISTRY_TLS="feast-$FS_NAME-registry-tls" + +cat >"$TMP/features.py" <<'PY' +from datetime import timedelta +from feast import Entity, FeatureService, FeatureView, Field, FileSource +from feast.data_format import ParquetFormat +from feast.types import Float32, Int64 +from feast.value_type import ValueType + +driver = Entity(name="driver", join_keys=["driver_id"], value_type=ValueType.INT64) +source = FileSource(name="driver_stats_source", path="data/driver_stats.parquet", + file_format=ParquetFormat(), timestamp_field="event_timestamp", + created_timestamp_column="created") +view = FeatureView(name="driver_hourly_stats", entities=[driver], ttl=timedelta(days=365), + schema=[Field(name="conv_rate", dtype=Float32), Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64)], online=True, source=source) +driver_activity_v1 = FeatureService(name="driver_activity_v1", features=[view]) +PY + +# The remote online-store client intentionally has a no-op infrastructure update +# in Feast 0.61. Run apply once in the operand's local repository so its SQLite +# table exists before a remote materialize call writes rows to the online server. +online_pod="" +for _ in $(seq 1 60); do + online_pod="$(feast_kc -n "$NS" get pods -l "feast.dev/name=$FS_NAME" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + [ -n "$online_pod" ] && break + sleep 5 +done +[ -n "$online_pod" ] || { log "Feast online pod did not appear"; exit 1; } +feast_kc -n "$NS" wait --for=condition=Ready "pod/$online_pod" --timeout=300s >/dev/null +feast_kc -n "$NS" cp "$TMP/features.py" \ + "$NS/$online_pod:/feast-data/$PROJECT/feature_repo/feature_definitions.py" -c online +feast_kc -n "$NS" exec "$online_pod" -c online -- \ + bash -c "cd /feast-data/$PROJECT/feature_repo && feast apply" + +cat >"$TMP/run.sh" <<'RUN' +#!/usr/bin/env bash +set -euo pipefail +R=/mnt/models/repo +mkdir -p "$R/data" +cp /etc/feast/feature_store.yaml "$R/feature_store.yaml" +cp /runner/features.py "$R/features.py" +python - <<'PY' +import numpy as np, pandas as pd +rng=np.random.default_rng(7); n=240 +df=pd.DataFrame({"driver_id":(np.arange(n)%12+1).astype("int64"), + "event_timestamp":pd.date_range("2026-01-01",periods=n,freq="h",tz="UTC")}) +df["created"]=df.event_timestamp+pd.to_timedelta(1,unit="m") +df["conv_rate"]=(.25+.55*rng.random(n)).astype("float32") +df["acc_rate"]=(.50+.45*rng.random(n)).astype("float32") +df["avg_daily_trips"]=rng.integers(2,20,size=n).astype("int64") +df["label"]=((df.conv_rate*2+df.acc_rate+df.avg_daily_trips/20)>1.8).astype("int64") +df.to_parquet("/mnt/models/repo/data/driver_stats.parquet",index=False) +PY +feast --chdir "$R" apply +python - <<'PY' +import numpy as np, pandas as pd +from feast import FeatureStore +r="/mnt/models/repo"; s=FeatureStore(repo_path=r); raw=pd.read_parquet(r+"/data/driver_stats.parquet") +t=s.get_historical_features(entity_df=raw[["driver_id","event_timestamp","label"]], + features=["driver_hourly_stats:conv_rate","driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips"]).to_df().dropna() +x=t[["conv_rate","acc_rate","avg_daily_trips"]].to_numpy(float); y=t.label.to_numpy(float) +w=np.linalg.pinv(np.column_stack([np.ones(len(x)),x]))@y +np.savez("/mnt/models/model.npz",weights=w) +s.materialize_incremental(raw.event_timestamp.max().to_pydatetime()+pd.Timedelta(hours=1)) +print("historical_rows",len(t),"weights",w.tolist()) +PY +cp "$R/feature_store.yaml" /mnt/models/feature_store.yaml +cp "$R/features.py" /mnt/models/features.py +cp /runner/server.py /mnt/models/server.py +RUN +cat >"$TMP/server.py" <<'PY' +import os, numpy as np, uvicorn +from fastapi import Body, FastAPI +from feast import FeatureStore +name=os.getenv("MODEL_NAME","feast-online-model") +w=np.load("/mnt/models/model.npz")["weights"]; s=FeatureStore(repo_path="/mnt/models") +fs=s.get_feature_service("driver_activity_v1"); app=FastAPI() +@app.get("/v2/health/ready") +@app.get("/v2/health/live") +def health(): return {"ready":True} +@app.get("/v2/models/{model_name}") +@app.get("/v2/models/{model_name}/ready") +def ready(model_name): return {"name":model_name,"ready":model_name==name} +@app.post("/v2/models/{model_name}/infer") +def infer(model_name, payload: dict = Body(...)): + ids=next(x for x in payload["inputs"] if x["name"]=="driver_id")["data"] + values=s.get_online_features(features=fs,entity_rows=[{"driver_id":int(x)} for x in ids]).to_dict() + def col(n): + return values[n] if n in values else values[next(k for k in values if k.endswith("__"+n))] + x=np.column_stack([np.ones(len(ids)),col("conv_rate"),col("acc_rate"),col("avg_daily_trips")]) + return {"model_name":model_name,"outputs":[{"name":"prediction","shape":[len(ids)],"datatype":"FP32","data":(x@w).astype("float32").tolist()}]} +if __name__=="__main__": uvicorn.run(app,host="0.0.0.0",port=8080) +PY + +cat </dev/null + +cat </dev/null || true)" + [ "${available:-0}" -ge 1 ] 2>/dev/null && break + sleep 10 +done +[ "${available:-0}" -ge 1 ] 2>/dev/null || { + log "KServe predictor deployment did not become available" + feast_kc -n "$NS" get inferenceservice "$ISVC" -o yaml >&2 || true + exit 1 +} +SVC="$ISVC-predictor" +feast_kc -n "$NS" get service "$SVC" >/dev/null +feast_kc -n "$NS" port-forward "service/$SVC" 18080:80 >"$TMP/pf.log" 2>&1 & +PF=$! +for _ in $(seq 1 30); do curl -fsS http://127.0.0.1:18080/v2/health/ready >/dev/null 2>&1 && break; sleep 2; done +response="$(curl -fsS -X POST "http://127.0.0.1:18080/v2/models/$ISVC/infer" -H 'Content-Type: application/json' -d '{"inputs":[{"name":"driver_id","shape":[2],"datatype":"INT64","data":[1,2]}]}')" +echo "$response"; echo "$response" | grep -q prediction +log "C16: Feast offline-to-online inference demo passed" diff --git a/e2e/lib.sh b/e2e/lib.sh index 52fa826f..b406c99f 100644 --- a/e2e/lib.sh +++ b/e2e/lib.sh @@ -18,7 +18,7 @@ case "${E2E_SKIP_RC}" in *) [ "${E2E_SKIP_RC}" -ge 0 ] && [ "${E2E_SKIP_RC}" -le 255 ] || E2E_SKIP_RC=77 ;; esac -# Required per case: GPU_NAMESPACE or NPU_NAMESPACE. +# Required per case: GPU_NAMESPACE, NPU_NAMESPACE, or FEAST_NAMESPACE. # Optional kube target: GPU_CONTEXT/GPU_KUBECONFIG/NPU_CONTEXT/NPU_KUBECONFIG. # Optional Docker Hub mirrors: GPU_DH_MIRROR/NPU_DH_MIRROR. # Optional private registry access: E2E_IMAGE_PULL_SECRET. @@ -31,6 +31,12 @@ NPU_KUBECONFIG="${NPU_KUBECONFIG:-}" NPU_NAMESPACE="${NPU_NAMESPACE:-}" GPU_DH_MIRROR="${GPU_DH_MIRROR:-}" NPU_DH_MIRROR="${NPU_DH_MIRROR:-}" +FEAST_CONTEXT="${FEAST_CONTEXT:-}" +FEAST_KUBECONFIG="${FEAST_KUBECONFIG:-}" +FEAST_NAMESPACE="${FEAST_NAMESPACE:-}" +FEAST_IMAGE="${FEAST_IMAGE:-}" +FEAST_STORAGE_CLASS="${FEAST_STORAGE_CLASS:-}" +FEAST_KEEP_RESOURCES="${FEAST_KEEP_RESOURCES:-0}" # Rewrite docker.io references to a mirror that the cluster can actually reach. # Args: mirror_host. Reads stdin, writes patched YAML to stdout. @@ -81,6 +87,7 @@ _kubectl_with_env() { gpu_kc() { _kubectl_with_env "${GPU_KUBECONFIG}" "${GPU_CONTEXT}" "$@"; } npu_kc() { _kubectl_with_env "${NPU_KUBECONFIG}" "${NPU_CONTEXT}" "$@"; } +feast_kc() { _kubectl_with_env "${FEAST_KUBECONFIG}" "${FEAST_CONTEXT}" "$@"; } yaml_scalar_field() { local indent="$1" name="$2" value="${3:-}" diff --git a/e2e/run_all.sh b/e2e/run_all.sh index 64fe1943..e2fe35db 100755 --- a/e2e/run_all.sh +++ b/e2e/run_all.sh @@ -5,6 +5,7 @@ # ./run_all.sh C1 C7 # run only the named cases # SKIP_NPU=1 ./run_all.sh # skip cases marked NPU # SKIP_GPU=1 ./run_all.sh # skip cases marked GPU +# SKIP_FEAST=1 ./run_all.sh # skip the Feast case set -uo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" @@ -41,6 +42,7 @@ CASES=( # (A30) is reserved by the persistent inference workload and no slice frees up; # the orchestrator controls A30 capacity. Same build-harbor image as C13. "C14:GPU:cases/c14_traininghub_cpt.sh" + "C16:FEAST:cases/c16_feast_offline_online.sh" ) want=( "$@" ) @@ -57,6 +59,7 @@ for entry in "${CASES[@]}"; do should_run "${id}" || { skip=$((skip+1)); continue; } if [ "${cluster}" = "GPU" ] && [ "${SKIP_GPU:-0}" = "1" ]; then skip=$((skip+1)); continue; fi if [ "${cluster}" = "NPU" ] && [ "${SKIP_NPU:-0}" = "1" ]; then skip=$((skip+1)); continue; fi + if [ "${cluster}" = "FEAST" ] && [ "${SKIP_FEAST:-0}" = "1" ]; then skip=$((skip+1)); continue; fi log_file="${LOG_DIR}/${id}.log" log "==> ${id} [${cluster}] -> ${log_file}" From 10ef9042641e985e7e0773c3374ce20333c926e0 Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Wed, 19 Aug 2026 10:57:36 +0800 Subject: [PATCH 2/7] docs: resolve Feast image from cluster registry --- .../feast-offline-to-online-inference.ipynb | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/en/train/guides/feast-offline-to-online-inference.ipynb b/docs/en/train/guides/feast-offline-to-online-inference.ipynb index 68082f9f..ed97caa0 100644 --- a/docs/en/train/guides/feast-offline-to-online-inference.ipynb +++ b/docs/en/train/guides/feast-offline-to-online-inference.ipynb @@ -14,7 +14,15 @@ "4. materialize the same features into the Feast online store; and\n", "5. deploy a KServe `InferenceService` whose model server reads the online features before predicting.\n", "\n", - "The notebook assumes `kubectl`, Feast 0.61.x, NumPy, pandas, PyArrow, `requests`, and KServe are available. It uses a PVC for the model artifact and the published Feast feature-server image for the custom serving runtime. The same flow is automated by [`e2e/cases/c16_feast_offline_online.sh`](../../../e2e/cases/c16_feast_offline_online.sh)." + "The notebook assumes `kubectl`, Feast 0.61.x, NumPy, pandas, PyArrow, `requests`, and KServe are available. It uses a PVC for the model artifact and the published Feast feature-server image for the custom serving runtime.\n", + "\n", + "The notebook discovers the cluster registry from `kube-public/global-info`. To inspect the registry address yourself, run:\n", + "\n", + "```bash\n", + "kubectl get configmap global-info -n kube-public -o jsonpath='{.data.registryAddress}'\n", + "```\n", + "\n", + "The default runtime image appends `mlops/feast/feature-server:0.61.0` to that address. Set `FEAST_MODEL_IMAGE` to a complete image reference if your environment uses a different repository or tag." ] }, { @@ -41,7 +49,6 @@ "MODEL_PVC = os.environ.get(\"FEAST_MODEL_PVC\", \"feast-notebook-model\")\n", "MODEL_RUNTIME = os.environ.get(\"FEAST_MODEL_RUNTIME\", \"feast-numpy-runtime\")\n", "MODEL_NAME = os.environ.get(\"FEAST_MODEL_NAME\", \"feast-online-model\")\n", - "MODEL_IMAGE = os.environ.get(\"FEAST_MODEL_IMAGE\", \"build-harbor.alauda.cn/mlops/feast/feature-server:0.61.0\")\n", "REPO = Path(\"feast-notebook-repo\")\n", "DATA_DIR = REPO / \"data\"\n", "MODEL_DIR = Path(\"feast-notebook-model\")\n", @@ -55,6 +62,13 @@ " raise RuntimeError(f\"kubectl {' '.join(args)} failed: {result.stderr}\")\n", " return result.stdout.strip()\n", "\n", + "MODEL_IMAGE = os.environ.get(\"FEAST_MODEL_IMAGE\")\n", + "if not MODEL_IMAGE:\n", + " registry_address = kubectl(\"get\", \"configmap\", \"global-info\", \"-n\", \"kube-public\", \"-o\", \"jsonpath={.data.registryAddress}\")\n", + " if not registry_address:\n", + " raise RuntimeError(\"kube-public/global-info does not contain data.registryAddress\")\n", + " MODEL_IMAGE = f\"{registry_address}/mlops/feast/feature-server:0.61.0\"\n", + "\n", "print({\"namespace\": NAMESPACE, \"featurestore\": FEATURESTORE_NAME, \"model_pvc\": MODEL_PVC})" ] }, @@ -114,7 +128,7 @@ "source": [ "## 2. Prepare a Parquet batch source\n", "\n", - "The event timestamp is required by Feast for point-in-time historical retrieval. The label is kept in the entity dataframe for model training and is not registered as a feature." + "This section generates a small synthetic batch dataset so the example is self-contained. The event timestamp is required by Feast for point-in-time historical retrieval. The label is kept in the entity dataframe for model training and is not registered as a feature." ] }, { From 4bf6500973376bb8f630fb70e68cdc6972705e28 Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Wed, 19 Aug 2026 12:20:49 +0800 Subject: [PATCH 3/7] docs: use durable Feast backends in notebook --- .../feast-offline-to-online-inference.ipynb | 139 +++++++++++++----- 1 file changed, 103 insertions(+), 36 deletions(-) diff --git a/docs/en/train/guides/feast-offline-to-online-inference.ipynb b/docs/en/train/guides/feast-offline-to-online-inference.ipynb index ed97caa0..478a66fc 100644 --- a/docs/en/train/guides/feast-offline-to-online-inference.ipynb +++ b/docs/en/train/guides/feast-offline-to-online-inference.ipynb @@ -8,13 +8,17 @@ "\n", "This notebook follows the [Feast quickstart](../../develop/components/feast/quickstart.mdx) and runs a small, CPU-only demonstration:\n", "\n", - "1. create a Parquet batch dataset;\n", + "1. create a synthetic batch dataset in PostgreSQL and archive it as Parquet in S3-compatible object storage;\n", "2. define and register Feast entities, feature views, and a feature service;\n", "3. retrieve historical features and train a sample NumPy model;\n", "4. materialize the same features into the Feast online store; and\n", "5. deploy a KServe `InferenceService` whose model server reads the online features before predicting.\n", "\n", - "The notebook assumes `kubectl`, Feast 0.61.x, NumPy, pandas, PyArrow, `requests`, and KServe are available. It uses a PVC for the model artifact and the published Feast feature-server image for the custom serving runtime.\n", + "The notebook assumes `kubectl`, Feast 0.61.x with the PostgreSQL and Redis extras, NumPy, pandas, PyArrow, boto3, SQLAlchemy, psycopg, PyYAML, `requests`, and KServe are available. It uses PostgreSQL for offline feature queries and the SQL registry, Redis for online serving, SeaweedFS or another S3-compatible service for durable Parquet dataset snapshots, and a PVC for the model artifact.\n", + "\n", + "> **Production note:** Feast classifies its PostgreSQL offline store as a contributed integration without full stability guarantees. S3 `FileSource` is also intended for development rather than high-scale serving. This notebook therefore queries features from PostgreSQL and uses S3 for versioned dataset snapshots. For larger production workloads, use a fully supported warehouse or distributed query engine, managed PostgreSQL and Redis with high availability, encrypted connections, secret rotation, backups, monitoring, and a scheduled materialization job.\n", + "\n", + "Before running the notebook, create an operator backend Secret named `feast-data-stores` with `postgres`, `redis`, and `sql` keys as described in the Feast quickstart. Also create the target object-storage bucket and a `feast-s3-credentials` Secret containing `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `S3_ENDPOINT_URL`, and `S3_BUCKET`. The notebook reads these existing Secrets without printing their values. In a long-lived Workbench, mount the Secrets as files and environment variables instead of granting broad Secret-read permissions.\n", "\n", "The notebook discovers the cluster registry from `kube-public/global-info`. To inspect the registry address yourself, run:\n", "\n", @@ -32,6 +36,7 @@ "outputs": [], "source": [ "import base64\n", + "import io\n", "import json\n", "import os\n", "import subprocess\n", @@ -42,6 +47,9 @@ "import numpy as np\n", "import pandas as pd\n", "import requests\n", + "import boto3\n", + "import yaml\n", + "from sqlalchemy import URL, create_engine\n", "\n", "NAMESPACE = os.environ.get(\"FEAST_NAMESPACE\", \"feast-demo\")\n", "FEATURESTORE_NAME = os.environ.get(\"FEAST_FEATURESTORE\", \"feast-notebook\")\n", @@ -49,11 +57,14 @@ "MODEL_PVC = os.environ.get(\"FEAST_MODEL_PVC\", \"feast-notebook-model\")\n", "MODEL_RUNTIME = os.environ.get(\"FEAST_MODEL_RUNTIME\", \"feast-numpy-runtime\")\n", "MODEL_NAME = os.environ.get(\"FEAST_MODEL_NAME\", \"feast-online-model\")\n", + "DATA_STORES_SECRET = os.environ.get(\"FEAST_DATA_STORES_SECRET\", \"feast-data-stores\")\n", + "S3_CREDENTIALS_SECRET = os.environ.get(\"FEAST_S3_CREDENTIALS_SECRET\", \"feast-s3-credentials\")\n", + "POSTGRES_SCHEMA = os.environ.get(\"FEAST_POSTGRES_SCHEMA\", \"public\")\n", + "POSTGRES_TABLE = os.environ.get(\"FEAST_POSTGRES_TABLE\", \"driver_stats\")\n", + "S3_DATASET_KEY = os.environ.get(\"FEAST_S3_DATASET_KEY\", \"datasets/driver_stats.parquet\")\n", "REPO = Path(\"feast-notebook-repo\")\n", - "DATA_DIR = REPO / \"data\"\n", "MODEL_DIR = Path(\"feast-notebook-model\")\n", "REPO.mkdir(exist_ok=True)\n", - "DATA_DIR.mkdir(parents=True, exist_ok=True)\n", "MODEL_DIR.mkdir(exist_ok=True)\n", "\n", "def kubectl(*args, input_text=None, check=True):\n", @@ -62,6 +73,16 @@ " raise RuntimeError(f\"kubectl {' '.join(args)} failed: {result.stderr}\")\n", " return result.stdout.strip()\n", "\n", + "def secret_value(secret_name, key):\n", + " encoded = kubectl(\"get\", \"secret\", secret_name, \"-n\", NAMESPACE, \"-o\", f\"jsonpath={{.data.{key}}}\")\n", + " if not encoded:\n", + " raise RuntimeError(f\"Secret {NAMESPACE}/{secret_name} does not contain {key}\")\n", + " return base64.b64decode(encoded).decode()\n", + "\n", + "for name, value in [(\"FEAST_POSTGRES_SCHEMA\", POSTGRES_SCHEMA), (\"FEAST_POSTGRES_TABLE\", POSTGRES_TABLE)]:\n", + " if not value.replace(\"_\", \"\").isalnum():\n", + " raise ValueError(f\"{name} must contain only letters, numbers, and underscores\")\n", + "\n", "MODEL_IMAGE = os.environ.get(\"FEAST_MODEL_IMAGE\")\n", "if not MODEL_IMAGE:\n", " registry_address = kubectl(\"get\", \"configmap\", \"global-info\", \"-n\", \"kube-public\", \"-o\", \"jsonpath={.data.registryAddress}\")\n", @@ -78,7 +99,7 @@ "source": [ "## 1. Prepare the FeatureStore operand\n", "\n", - "The quickstart requires `registry.local.server: {}` when a client outside the Feast pod runs `feast apply`. The UI is optional for this workflow, but is enabled here for inspection. If you already have a suitable `FeatureStore`, set `FEAST_FEATURESTORE` and skip this cell." + "This profile uses the existing `feast-data-stores` Secret for three operator-managed backends: PostgreSQL offline queries, a PostgreSQL SQL registry, and Redis online serving. The Secret must exist before the `FeatureStore` is created. The UI is optional for this workflow, but is enabled here for inspection. If you already have a suitable `FeatureStore`, set `FEAST_FEATURESTORE` and skip this cell." ] }, { @@ -98,9 +119,28 @@ " namespace: {NAMESPACE}\n", "spec:\n", " feastProject: {FEAST_PROJECT}\n", + " replicas: 1\n", " services:\n", + " offlineStore:\n", + " persistence:\n", + " store:\n", + " type: postgres\n", + " secretRef:\n", + " name: {DATA_STORES_SECRET}\n", + " server: {{}}\n", + " onlineStore:\n", + " persistence:\n", + " store:\n", + " type: redis\n", + " secretRef:\n", + " name: {DATA_STORES_SECRET}\n", " registry:\n", " local:\n", + " persistence:\n", + " store:\n", + " type: sql\n", + " secretRef:\n", + " name: {DATA_STORES_SECRET}\n", " server: {{}}\n", " ui: {{}}\n", "\"\"\"\n", @@ -126,9 +166,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 2. Prepare a Parquet batch source\n", + "## 2. Prepare a synthetic dataset in PostgreSQL and S3\n", "\n", - "This section generates a small synthetic batch dataset so the example is self-contained. The event timestamp is required by Feast for point-in-time historical retrieval. The label is kept in the entity dataframe for model training and is not registered as a feature." + "This section generates a small synthetic batch dataset so the example is self-contained. It loads the queryable feature table into PostgreSQL and writes the same rows as a versionable Parquet snapshot in S3-compatible object storage. Feast reads the PostgreSQL table for historical retrieval; the S3 object is the durable dataset artifact. The event timestamp is required for point-in-time retrieval, while the label is retained for model training and is not registered as a feature." ] }, { @@ -141,16 +181,42 @@ "n_rows = 240\n", "events = pd.DataFrame({\n", " \"driver_id\": (np.arange(n_rows) % 12 + 1).astype(\"int64\"),\n", - " \"event_timestamp\": pd.date_range(\"2026-01-01\", periods=n_rows, freq=\"h\", tz=\"UTC\"),\n", + " \"event_timestamp\": pd.date_range(end=pd.Timestamp.now(tz=\"UTC\").floor(\"h\"), periods=n_rows, freq=\"h\"),\n", "})\n", "events[\"created\"] = events[\"event_timestamp\"] + pd.to_timedelta(1, unit=\"m\")\n", "events[\"conv_rate\"] = (0.25 + 0.55 * rng.random(n_rows)).astype(\"float32\")\n", "events[\"acc_rate\"] = (0.50 + 0.45 * rng.random(n_rows)).astype(\"float32\")\n", "events[\"avg_daily_trips\"] = rng.integers(2, 20, size=n_rows).astype(\"int64\")\n", "events[\"label\"] = ((events[\"conv_rate\"] * 2 + events[\"acc_rate\"] + events[\"avg_daily_trips\"] / 20) > 1.8).astype(\"int64\")\n", - "parquet_path = DATA_DIR / \"driver_stats.parquet\"\n", - "events.to_parquet(parquet_path, index=False)\n", - "print(parquet_path, events.shape)\n", + "postgres_config = yaml.safe_load(secret_value(DATA_STORES_SECRET, \"postgres\"))\n", + "postgres_url = URL.create(\n", + " \"postgresql+psycopg\",\n", + " username=postgres_config[\"user\"],\n", + " password=postgres_config[\"password\"],\n", + " host=postgres_config[\"host\"],\n", + " port=postgres_config[\"port\"],\n", + " database=postgres_config[\"database\"],\n", + ")\n", + "engine = create_engine(postgres_url, pool_pre_ping=True)\n", + "events.to_sql(POSTGRES_TABLE, engine, schema=POSTGRES_SCHEMA,\n", + " if_exists=\"replace\", index=False, method=\"multi\")\n", + "\n", + "s3_environment = {\n", + " key: os.environ.get(key) or secret_value(S3_CREDENTIALS_SECRET, key)\n", + " for key in [\"AWS_ACCESS_KEY_ID\", \"AWS_SECRET_ACCESS_KEY\", \"AWS_DEFAULT_REGION\",\n", + " \"S3_ENDPOINT_URL\", \"S3_BUCKET\"]\n", + "}\n", + "s3 = boto3.client(\n", + " \"s3\", endpoint_url=s3_environment[\"S3_ENDPOINT_URL\"],\n", + " aws_access_key_id=s3_environment[\"AWS_ACCESS_KEY_ID\"],\n", + " aws_secret_access_key=s3_environment[\"AWS_SECRET_ACCESS_KEY\"],\n", + " region_name=s3_environment[\"AWS_DEFAULT_REGION\"],\n", + ")\n", + "parquet_buffer = io.BytesIO()\n", + "events.to_parquet(parquet_buffer, index=False)\n", + "s3.put_object(Bucket=s3_environment[\"S3_BUCKET\"], Key=S3_DATASET_KEY, Body=parquet_buffer.getvalue())\n", + "print({\"rows\": len(events), \"postgres_table\": POSTGRES_TABLE,\n", + " \"s3_uri\": f\"s3://{s3_environment['S3_BUCKET']}/{S3_DATASET_KEY}\"})\n", "events.head()" ] }, @@ -160,7 +226,7 @@ "source": [ "## 3. Define and register offline features\n", "\n", - "This follows the quickstart’s `Entity` + `FileSource` + `FeatureView` + `FeatureService` pattern. For the operand’s default PVC-backed SQLite online store, the cell also runs `feast apply` inside the online-store pod once, creating the feature-view table before remote materialization writes to it." + "This uses `PostgreSQLSource` for point-in-time historical queries. `feast apply` writes definitions to the PostgreSQL-backed SQL registry and prepares the Redis online-store infrastructure. The Parquet snapshot remains in S3 for reproducibility and downstream dataset consumers; it is not used as a development-only `FileSource`." ] }, { @@ -169,17 +235,18 @@ "metadata": {}, "outputs": [], "source": [ - "(REPO / \"features.py\").write_text(textwrap.dedent(\"\"\"\n", + "(REPO / \"features.py\").write_text(textwrap.dedent(f\"\"\"\n", " from datetime import timedelta\n", - " from feast import Entity, FeatureService, FeatureView, Field, FileSource\n", - " from feast.data_format import ParquetFormat\n", + " from feast import Entity, FeatureService, FeatureView, Field\n", + " from feast.infra.offline_stores.contrib.postgres_offline_store.postgres_source import PostgreSQLSource\n", " from feast.types import Float32, Int64\n", " from feast.value_type import ValueType\n", "\n", " driver = Entity(name=\"driver\", join_keys=[\"driver_id\"], value_type=ValueType.INT64)\n", - " driver_stats_source = FileSource(\n", - " name=\"driver_stats_source\", path=\"data/driver_stats.parquet\",\n", - " file_format=ParquetFormat(), timestamp_field=\"event_timestamp\",\n", + " driver_stats_source = PostgreSQLSource(\n", + " name=\"driver_stats_source\",\n", + " query=\"SELECT * FROM {POSTGRES_SCHEMA}.{POSTGRES_TABLE}\",\n", + " timestamp_field=\"event_timestamp\",\n", " created_timestamp_column=\"created\",\n", " )\n", " driver_hourly_stats = FeatureView(\n", @@ -199,6 +266,7 @@ "runtime_config = client_config\n", "local_config = runtime_config\n", "for secret_name, original_path, local_name in [\n", + " (f\"feast-{FEATURESTORE_NAME}-offline-tls\", \"/tls/offline/tls.crt\", \"offline-tls.crt\"),\n", " (f\"feast-{FEATURESTORE_NAME}-online-tls\", \"/tls/online/tls.crt\", \"online-tls.crt\"),\n", " (f\"feast-{FEATURESTORE_NAME}-registry-tls\", \"/tls/registry/tls.crt\", \"registry-tls.crt\"),\n", "]:\n", @@ -212,18 +280,7 @@ "(MODEL_DIR / \"features.py\").write_text((REPO / \"features.py\").read_text())\n", "subprocess.run([\"feast\", \"--chdir\", str(REPO), \"apply\"], check=True)\n", "\n", - "# Feast 0.61's remote online-store client does not create SQLite tables.\n", - "# Apply once in the operand's local repository to create that infrastructure.\n", - "online_pod = kubectl(\"get\", \"pods\", \"-n\", NAMESPACE, \"-l\", f\"feast.dev/name={FEATURESTORE_NAME}\",\n", - " \"-o\", \"jsonpath={.items[0].metadata.name}\")\n", - "if not online_pod:\n", - " raise RuntimeError(\"FeatureStore online pod was not found\")\n", - "kubectl(\"cp\", str(REPO / \"features.py\"),\n", - " f\"{NAMESPACE}/{online_pod}:/feast-data/{FEAST_PROJECT}/feature_repo/feature_definitions.py\",\n", - " \"-c\", \"online\")\n", - "kubectl(\"exec\", \"-n\", NAMESPACE, online_pod, \"-c\", \"online\", \"--\", \"bash\", \"-c\",\n", - " f\"cd /feast-data/{FEAST_PROJECT}/feature_repo && feast apply\")\n", - "print(\"Feature definitions registered; online-store infrastructure created\")" + "print(\"Feature definitions registered in the SQL registry; Redis infrastructure prepared\")" ] }, { @@ -232,7 +289,7 @@ "source": [ "## 4. Retrieve offline features and train a sample model\n", "\n", - "The model is deliberately a small NumPy linear classifier so the example does not require a second private training image. The training matrix comes from Feast’s historical feature retrieval, not directly from the raw Parquet columns." + "The model is deliberately a small NumPy linear classifier so the example does not require a second training image. The training matrix comes from Feast’s point-in-time PostgreSQL retrieval, not directly from the in-memory dataframe or S3 snapshot." ] }, { @@ -269,7 +326,7 @@ "source": [ "## 5. Materialize and verify online features\n", "\n", - "`materialize_incremental` copies the registered batch features into the online store. The inference server below uses the same feature service and entity key at request time." + "`materialize_incremental` copies the registered PostgreSQL features into Redis. The SQL registry cache may take up to its configured TTL to expose newly applied definitions to the online server, so the verification retries during that propagation window. The inference server below uses the same feature service and entity key at request time." ] }, { @@ -278,12 +335,22 @@ "metadata": {}, "outputs": [], "source": [ + "from feast.errors import FeatureViewNotFoundException\n", + "\n", "end_date = events[\"event_timestamp\"].max().to_pydatetime() + pd.Timedelta(hours=1)\n", "store.materialize_incremental(end_date)\n", - "online = store.get_online_features(\n", - " features=store.get_feature_service(\"driver_activity_v1\"),\n", - " entity_rows=[{\"driver_id\": 1}, {\"driver_id\": 2}],\n", - ").to_df()\n", + "deadline = time.time() + 90\n", + "while True:\n", + " try:\n", + " online = store.get_online_features(\n", + " features=store.get_feature_service(\"driver_activity_v1\"),\n", + " entity_rows=[{\"driver_id\": 1}, {\"driver_id\": 2}],\n", + " ).to_df()\n", + " break\n", + " except FeatureViewNotFoundException:\n", + " if time.time() >= deadline:\n", + " raise\n", + " time.sleep(5)\n", "online" ] }, From ae2dcfab4416938c336d44a6192f08852c356650 Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Wed, 19 Aug 2026 16:29:25 +0800 Subject: [PATCH 4/7] docs: run Feast offline batch with Spark Operator --- .../feast-offline-to-online-inference.ipynb | 719 ++++++++++-------- e2e/cases/c16_feast_offline_online.sh | 446 +++++++---- e2e/lib.sh | 6 + 3 files changed, 744 insertions(+), 427 deletions(-) diff --git a/docs/en/train/guides/feast-offline-to-online-inference.ipynb b/docs/en/train/guides/feast-offline-to-online-inference.ipynb index 478a66fc..2d89f69e 100644 --- a/docs/en/train/guides/feast-offline-to-online-inference.ipynb +++ b/docs/en/train/guides/feast-offline-to-online-inference.ipynb @@ -6,27 +6,31 @@ "source": [ "# Feast offline-to-online training and inference\n", "\n", - "This notebook follows the [Feast quickstart](../../develop/components/feast/quickstart.mdx) and runs a small, CPU-only demonstration:\n", + "This notebook runs a production-shaped, CPU-only feature pipeline:\n", "\n", - "1. create a synthetic batch dataset in PostgreSQL and archive it as Parquet in S3-compatible object storage;\n", - "2. define and register Feast entities, feature views, and a feature service;\n", - "3. retrieve historical features and train a sample NumPy model;\n", - "4. materialize the same features into the Feast online store; and\n", - "5. deploy a KServe `InferenceService` whose model server reads the online features before predicting.\n", + "1. a Spark Operator `SparkApplication` generates a synthetic batch dataset and stores it as Parquet in S3-compatible object storage;\n", + "2. Spark performs Feast's point-in-time historical join and trains a small NumPy model;\n", + "3. Feast uses PostgreSQL for its durable SQL registry and materializes the latest feature values into Redis;\n", + "4. KServe deploys a model server that reads Redis-backed online features before predicting.\n", "\n", - "The notebook assumes `kubectl`, Feast 0.61.x with the PostgreSQL and Redis extras, NumPy, pandas, PyArrow, boto3, SQLAlchemy, psycopg, PyYAML, `requests`, and KServe are available. It uses PostgreSQL for offline feature queries and the SQL registry, Redis for online serving, SeaweedFS or another S3-compatible service for durable Parquet dataset snapshots, and a PVC for the model artifact.\n", + "This separation is the recommended starting point for larger workloads: object storage is the durable, versionable offline data layer; operator-managed Spark provides elastic batch compute; PostgreSQL stores Feast metadata rather than the feature history; and Redis serves low-latency online lookups. Feast classifies its Spark offline store as a contributed integration without full test coverage, so qualify it against your scale and upgrade requirements or use a fully supported warehouse while retaining the same S3-and-Spark data pipeline. For production, use highly available PostgreSQL, Redis, and S3-compatible services with TLS, secret rotation, backups, monitoring, retention policies, and scheduled `SparkApplication` materialization runs.\n", "\n", - "> **Production note:** Feast classifies its PostgreSQL offline store as a contributed integration without full stability guarantees. S3 `FileSource` is also intended for development rather than high-scale serving. This notebook therefore queries features from PostgreSQL and uses S3 for versioned dataset snapshots. For larger production workloads, use a fully supported warehouse or distributed query engine, managed PostgreSQL and Redis with high availability, encrypted connections, secret rotation, backups, monitoring, and a scheduled materialization job.\n", + "Prerequisites:\n", "\n", - "Before running the notebook, create an operator backend Secret named `feast-data-stores` with `postgres`, `redis`, and `sql` keys as described in the Feast quickstart. Also create the target object-storage bucket and a `feast-s3-credentials` Secret containing `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `S3_ENDPOINT_URL`, and `S3_BUCKET`. The notebook reads these existing Secrets without printing their values. In a long-lived Workbench, mount the Secrets as files and environment variables instead of granting broad Secret-read permissions.\n", + "- the Alauda Spark Operator is installed through OLM and `sparkapplications.sparkoperator.k8s.io` exists;\n", + "- a Spark runtime image containing PySpark, Feast 0.61.x with Spark and Redis support, NumPy, pandas, PyArrow, PyYAML, and a Hadoop S3A connector compatible with the image's Hadoop version;\n", + "- a `feast-data-stores` Secret with `redis` and `sql` keys for the Feast Operator;\n", + "- a `feast-s3-credentials` Secret with `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `S3_ENDPOINT_URL`, and `S3_BUCKET`;\n", + "- a pre-created S3 bucket and a ReadWriteOnce storage class for the sample model PVC; and\n", + "- the Feast and KServe operators.\n", "\n", - "The notebook discovers the cluster registry from `kube-public/global-info`. To inspect the registry address yourself, run:\n", + "Do not copy an internal registry hostname from this document. Find the registry for your global cluster with:\n", "\n", "```bash\n", "kubectl get configmap global-info -n kube-public -o jsonpath='{.data.registryAddress}'\n", "```\n", "\n", - "The default runtime image appends `mlops/feast/feature-server:0.61.0` to that address. Set `FEAST_MODEL_IMAGE` to a complete image reference if your environment uses a different repository or tag." + "In the ACP console, open the container registry associated with that address, choose an approved Spark runtime that satisfies the dependency list above, and set its complete reference in `FEAST_SPARK_IMAGE`. Set `FEAST_MODEL_IMAGE` similarly if the default Feast model-server repository or tag is not mirrored in your global registry.\n" ] }, { @@ -35,21 +39,13 @@ "metadata": {}, "outputs": [], "source": [ - "import base64\n", - "import io\n", "import json\n", "import os\n", "import subprocess\n", - "import textwrap\n", "import time\n", "from pathlib import Path\n", "\n", - "import numpy as np\n", - "import pandas as pd\n", "import requests\n", - "import boto3\n", - "import yaml\n", - "from sqlalchemy import URL, create_engine\n", "\n", "NAMESPACE = os.environ.get(\"FEAST_NAMESPACE\", \"feast-demo\")\n", "FEATURESTORE_NAME = os.environ.get(\"FEAST_FEATURESTORE\", \"feast-notebook\")\n", @@ -57,49 +53,56 @@ "MODEL_PVC = os.environ.get(\"FEAST_MODEL_PVC\", \"feast-notebook-model\")\n", "MODEL_RUNTIME = os.environ.get(\"FEAST_MODEL_RUNTIME\", \"feast-numpy-runtime\")\n", "MODEL_NAME = os.environ.get(\"FEAST_MODEL_NAME\", \"feast-online-model\")\n", + "SPARK_APP = os.environ.get(\"FEAST_SPARK_APPLICATION\", \"feast-offline-batch\")\n", + "SPARK_SERVICE_ACCOUNT = os.environ.get(\"FEAST_SPARK_SERVICE_ACCOUNT\", \"feast-spark\")\n", + "SPARK_VERSION = os.environ.get(\"FEAST_SPARK_VERSION\", \"4.0.1\")\n", "DATA_STORES_SECRET = os.environ.get(\"FEAST_DATA_STORES_SECRET\", \"feast-data-stores\")\n", "S3_CREDENTIALS_SECRET = os.environ.get(\"FEAST_S3_CREDENTIALS_SECRET\", \"feast-s3-credentials\")\n", - "POSTGRES_SCHEMA = os.environ.get(\"FEAST_POSTGRES_SCHEMA\", \"public\")\n", - "POSTGRES_TABLE = os.environ.get(\"FEAST_POSTGRES_TABLE\", \"driver_stats\")\n", - "S3_DATASET_KEY = os.environ.get(\"FEAST_S3_DATASET_KEY\", \"datasets/driver_stats.parquet\")\n", - "REPO = Path(\"feast-notebook-repo\")\n", - "MODEL_DIR = Path(\"feast-notebook-model\")\n", - "REPO.mkdir(exist_ok=True)\n", - "MODEL_DIR.mkdir(exist_ok=True)\n", - "\n", + "S3_DATASET_KEY = os.environ.get(\"FEAST_S3_DATASET_KEY\", \"datasets/driver_stats\")\n", "def kubectl(*args, input_text=None, check=True):\n", " result = subprocess.run([\"kubectl\", *args], input=input_text, text=True, capture_output=True)\n", " if check and result.returncode:\n", " raise RuntimeError(f\"kubectl {' '.join(args)} failed: {result.stderr}\")\n", " return result.stdout.strip()\n", "\n", - "def secret_value(secret_name, key):\n", - " encoded = kubectl(\"get\", \"secret\", secret_name, \"-n\", NAMESPACE, \"-o\", f\"jsonpath={{.data.{key}}}\")\n", - " if not encoded:\n", - " raise RuntimeError(f\"Secret {NAMESPACE}/{secret_name} does not contain {key}\")\n", - " return base64.b64decode(encoded).decode()\n", + "def secret_exists(name):\n", + " return bool(kubectl(\"get\", \"secret\", name, \"-n\", NAMESPACE, \"-o\", \"name\", check=False))\n", "\n", - "for name, value in [(\"FEAST_POSTGRES_SCHEMA\", POSTGRES_SCHEMA), (\"FEAST_POSTGRES_TABLE\", POSTGRES_TABLE)]:\n", - " if not value.replace(\"_\", \"\").isalnum():\n", - " raise ValueError(f\"{name} must contain only letters, numbers, and underscores\")\n", + "registry_address = kubectl(\n", + " \"get\", \"configmap\", \"global-info\", \"-n\", \"kube-public\",\n", + " \"-o\", \"jsonpath={.data.registryAddress}\",\n", + ")\n", + "if not registry_address:\n", + " raise RuntimeError(\"kube-public/global-info does not contain data.registryAddress\")\n", + "\n", + "SPARK_IMAGE = os.environ.get(\"FEAST_SPARK_IMAGE\")\n", + "if not SPARK_IMAGE:\n", + " raise RuntimeError(\n", + " \"Set FEAST_SPARK_IMAGE to an approved Spark+Feast runtime from the registry \"\n", + " f\"reported by kube-public/global-info ({registry_address}).\"\n", + " )\n", + "MODEL_IMAGE = os.environ.get(\n", + " \"FEAST_MODEL_IMAGE\", f\"{registry_address}/mlops/feast/feature-server:0.61.0\"\n", + ")\n", "\n", - "MODEL_IMAGE = os.environ.get(\"FEAST_MODEL_IMAGE\")\n", - "if not MODEL_IMAGE:\n", - " registry_address = kubectl(\"get\", \"configmap\", \"global-info\", \"-n\", \"kube-public\", \"-o\", \"jsonpath={.data.registryAddress}\")\n", - " if not registry_address:\n", - " raise RuntimeError(\"kube-public/global-info does not contain data.registryAddress\")\n", - " MODEL_IMAGE = f\"{registry_address}/mlops/feast/feature-server:0.61.0\"\n", + "if not kubectl(\"get\", \"crd\", \"sparkapplications.sparkoperator.k8s.io\", \"-o\", \"name\", check=False):\n", + " raise RuntimeError(\"Install the OLM Spark Operator before continuing\")\n", "\n", - "print({\"namespace\": NAMESPACE, \"featurestore\": FEATURESTORE_NAME, \"model_pvc\": MODEL_PVC})" + "print({\n", + " \"namespace\": NAMESPACE,\n", + " \"featurestore\": FEATURESTORE_NAME,\n", + " \"spark_application\": SPARK_APP,\n", + " \"spark_registry\": registry_address,\n", + "})\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 1. Prepare the FeatureStore operand\n", + "## 1. Prepare PostgreSQL registry and Redis online serving\n", "\n", - "This profile uses the existing `feast-data-stores` Secret for three operator-managed backends: PostgreSQL offline queries, a PostgreSQL SQL registry, and Redis online serving. The Secret must exist before the `FeatureStore` is created. The UI is optional for this workflow, but is enabled here for inspection. If you already have a suitable `FeatureStore`, set `FEAST_FEATURESTORE` and skip this cell." + "The Feast Operator manages only the control plane and online-serving backends here. PostgreSQL stores the Feast SQL registry; Redis stores materialized online feature values. The Spark driver overrides the generated client configuration with Feast's Spark offline store, so no Feast offline-server pod or separate Spark/Hadoop installation is needed.\n" ] }, { @@ -111,6 +114,11 @@ "for namespace in (NAMESPACE, \"feast-operator-system\"):\n", " namespace_yaml = kubectl(\"create\", \"namespace\", namespace, \"--dry-run=client\", \"-o\", \"yaml\")\n", " kubectl(\"apply\", \"-f\", \"-\", input_text=namespace_yaml)\n", + "\n", + "for secret_name in (DATA_STORES_SECRET, S3_CREDENTIALS_SECRET):\n", + " if not secret_exists(secret_name):\n", + " raise RuntimeError(f\"Create Secret {NAMESPACE}/{secret_name} before continuing\")\n", + "\n", "featurestore_yaml = f\"\"\"\n", "apiVersion: feast.dev/v1\n", "kind: FeatureStore\n", @@ -121,13 +129,6 @@ " feastProject: {FEAST_PROJECT}\n", " replicas: 1\n", " services:\n", - " offlineStore:\n", - " persistence:\n", - " store:\n", - " type: postgres\n", - " secretRef:\n", - " name: {DATA_STORES_SECRET}\n", - " server: {{}}\n", " onlineStore:\n", " persistence:\n", " store:\n", @@ -145,9 +146,13 @@ " ui: {{}}\n", "\"\"\"\n", "kubectl(\"apply\", \"-f\", \"-\", input_text=featurestore_yaml)\n", + "\n", "deadline = time.time() + 600\n", "while time.time() < deadline:\n", - " phase = kubectl(\"get\", \"featurestore\", FEATURESTORE_NAME, \"-n\", NAMESPACE, \"-o\", \"jsonpath={.status.phase}\", check=False)\n", + " phase = kubectl(\n", + " \"get\", \"featurestore\", FEATURESTORE_NAME, \"-n\", NAMESPACE,\n", + " \"-o\", \"jsonpath={.status.phase}\", check=False,\n", + " )\n", " print(phase or \"Pending\")\n", " if phase == \"Ready\":\n", " break\n", @@ -157,18 +162,22 @@ "else:\n", " raise TimeoutError(\"FeatureStore did not become Ready\")\n", "\n", - "client_config_map = kubectl(\"get\", \"featurestore\", FEATURESTORE_NAME, \"-n\", NAMESPACE, \"-o\", \"jsonpath={.status.clientConfigMap}\")\n", - "client_config = kubectl(\"get\", \"configmap\", client_config_map, \"-n\", NAMESPACE, \"-o\", r\"jsonpath={.data.feature_store\\.yaml}\")\n", - "print(\"FeatureStore is Ready; client config length:\", len(client_config))" + "client_config_map = kubectl(\n", + " \"get\", \"featurestore\", FEATURESTORE_NAME, \"-n\", NAMESPACE,\n", + " \"-o\", \"jsonpath={.status.clientConfigMap}\",\n", + ")\n", + "print(\"FeatureStore is Ready; client ConfigMap:\", client_config_map)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 2. Prepare a synthetic dataset in PostgreSQL and S3\n", + "## 2. Prepare the synthetic S3 batch application\n", + "\n", + "This section creates the code that the Spark Operator will mount into the driver. The application generates deterministic synthetic driver events with Spark, writes partitioned Parquet to the configured S3-compatible service, registers a Feast `SparkSource`, performs the point-in-time historical join, trains the sample model, and materializes the same features into Redis.\n", "\n", - "This section generates a small synthetic batch dataset so the example is self-contained. It loads the queryable feature table into PostgreSQL and writes the same rows as a versionable Parquet snapshot in S3-compatible object storage. Feast reads the PostgreSQL table for historical retrieval; the S3 object is the durable dataset artifact. The event timestamp is required for point-in-time retrieval, while the label is retained for model training and is not registered as a feature." + "The S3 endpoint and credentials come from a Kubernetes Secret. The code enables path-style access so it works with SeaweedFS and similar S3-compatible services; use your service's TLS endpoint in production.\n" ] }, { @@ -177,131 +186,102 @@ "metadata": {}, "outputs": [], "source": [ - "rng = np.random.default_rng(7)\n", + "batch_source = r'''import copy\n", + "import json\n", + "import os\n", + "import shutil\n", + "import subprocess\n", + "from pathlib import Path\n", + "from urllib.parse import urlparse\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import yaml\n", + "from feast import FeatureStore\n", + "from pyspark.sql import SparkSession, functions as F\n", + "\n", + "project = os.environ[\"FEAST_PROJECT\"]\n", + "bucket = os.environ[\"S3_BUCKET\"]\n", + "dataset_key = os.environ[\"S3_DATASET_KEY\"].strip(\"/\")\n", + "dataset_uri = f\"s3a://{bucket}/{dataset_key}\"\n", + "region = os.environ[\"AWS_DEFAULT_REGION\"]\n", + "repo = Path(\"/tmp/feast-repo\")\n", + "model_repo = Path(\"/mnt/models/repo\")\n", + "repo.mkdir(parents=True, exist_ok=True)\n", + "model_repo.mkdir(parents=True, exist_ok=True)\n", + "\n", + "spark = SparkSession.builder.appName(\"feast-offline-batch\").getOrCreate()\n", + "endpoint = urlparse(os.environ[\"S3_ENDPOINT_URL\"])\n", + "hadoop = spark.sparkContext._jsc.hadoopConfiguration()\n", + "hadoop.set(\"fs.s3a.endpoint\", endpoint.netloc or endpoint.path)\n", + "hadoop.set(\"fs.s3a.endpoint.region\", region)\n", + "hadoop.set(\"fs.s3a.path.style.access\", \"true\")\n", + "hadoop.set(\"fs.s3a.connection.ssl.enabled\", str(endpoint.scheme == \"https\").lower())\n", + "hadoop.set(\"fs.s3a.access.key\", os.environ[\"AWS_ACCESS_KEY_ID\"])\n", + "hadoop.set(\"fs.s3a.secret.key\", os.environ[\"AWS_SECRET_ACCESS_KEY\"])\n", + "\n", "n_rows = 240\n", - "events = pd.DataFrame({\n", - " \"driver_id\": (np.arange(n_rows) % 12 + 1).astype(\"int64\"),\n", - " \"event_timestamp\": pd.date_range(end=pd.Timestamp.now(tz=\"UTC\").floor(\"h\"), periods=n_rows, freq=\"h\"),\n", - "})\n", - "events[\"created\"] = events[\"event_timestamp\"] + pd.to_timedelta(1, unit=\"m\")\n", - "events[\"conv_rate\"] = (0.25 + 0.55 * rng.random(n_rows)).astype(\"float32\")\n", - "events[\"acc_rate\"] = (0.50 + 0.45 * rng.random(n_rows)).astype(\"float32\")\n", - "events[\"avg_daily_trips\"] = rng.integers(2, 20, size=n_rows).astype(\"int64\")\n", - "events[\"label\"] = ((events[\"conv_rate\"] * 2 + events[\"acc_rate\"] + events[\"avg_daily_trips\"] / 20) > 1.8).astype(\"int64\")\n", - "postgres_config = yaml.safe_load(secret_value(DATA_STORES_SECRET, \"postgres\"))\n", - "postgres_url = URL.create(\n", - " \"postgresql+psycopg\",\n", - " username=postgres_config[\"user\"],\n", - " password=postgres_config[\"password\"],\n", - " host=postgres_config[\"host\"],\n", - " port=postgres_config[\"port\"],\n", - " database=postgres_config[\"database\"],\n", + "events = (\n", + " spark.range(n_rows)\n", + " .withColumn(\"driver_id\", (F.col(\"id\") % 12 + 1).cast(\"long\"))\n", + " .withColumn(\"event_timestamp\", F.timestamp_seconds(F.lit(1767225600) + F.col(\"id\") * 3600))\n", + " .withColumn(\"created\", F.col(\"event_timestamp\") + F.expr(\"INTERVAL 1 MINUTE\"))\n", + " .withColumn(\"conv_rate\", (F.lit(0.25) + F.lit(0.55) * F.rand(7)).cast(\"float\"))\n", + " .withColumn(\"acc_rate\", (F.lit(0.50) + F.lit(0.45) * F.rand(11)).cast(\"float\"))\n", + " .withColumn(\"avg_daily_trips\", F.floor(F.lit(2) + F.lit(18) * F.rand(13)).cast(\"long\"))\n", + " .withColumn(\n", + " \"label\",\n", + " ((F.col(\"conv_rate\") * 2 + F.col(\"acc_rate\") + F.col(\"avg_daily_trips\") / 20) > 1.8).cast(\"long\"),\n", + " )\n", + " .drop(\"id\")\n", ")\n", - "engine = create_engine(postgres_url, pool_pre_ping=True)\n", - "events.to_sql(POSTGRES_TABLE, engine, schema=POSTGRES_SCHEMA,\n", - " if_exists=\"replace\", index=False, method=\"multi\")\n", - "\n", - "s3_environment = {\n", - " key: os.environ.get(key) or secret_value(S3_CREDENTIALS_SECRET, key)\n", - " for key in [\"AWS_ACCESS_KEY_ID\", \"AWS_SECRET_ACCESS_KEY\", \"AWS_DEFAULT_REGION\",\n", - " \"S3_ENDPOINT_URL\", \"S3_BUCKET\"]\n", + "events.repartition(4, \"driver_id\").write.mode(\"overwrite\").partitionBy(\"driver_id\").parquet(dataset_uri)\n", + "\n", + "client_config = yaml.safe_load(Path(\"/etc/feast/feature_store.yaml\").read_text())\n", + "batch_config = copy.deepcopy(client_config)\n", + "batch_config[\"offline_store\"] = {\n", + " \"type\": \"spark\",\n", + " \"spark_conf\": {\n", + " \"spark.sql.session.timeZone\": \"UTC\",\n", + " \"spark.hadoop.fs.s3a.endpoint\": endpoint.netloc or endpoint.path,\n", + " \"spark.hadoop.fs.s3a.endpoint.region\": region,\n", + " \"spark.hadoop.fs.s3a.path.style.access\": \"true\",\n", + " \"spark.hadoop.fs.s3a.connection.ssl.enabled\": str(endpoint.scheme == \"https\").lower(),\n", + " },\n", "}\n", - "s3 = boto3.client(\n", - " \"s3\", endpoint_url=s3_environment[\"S3_ENDPOINT_URL\"],\n", - " aws_access_key_id=s3_environment[\"AWS_ACCESS_KEY_ID\"],\n", - " aws_secret_access_key=s3_environment[\"AWS_SECRET_ACCESS_KEY\"],\n", - " region_name=s3_environment[\"AWS_DEFAULT_REGION\"],\n", + "(repo / \"feature_store.yaml\").write_text(yaml.safe_dump(batch_config, sort_keys=False))\n", + "(repo / \"features.py\").write_text(f\"\"\"from datetime import timedelta\n", + "from feast import Entity, FeatureService, FeatureView, Field\n", + "from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import SparkSource\n", + "from feast.types import Float32, Int64\n", + "from feast.value_type import ValueType\n", + "\n", + "driver = Entity(name=\"driver\", join_keys=[\"driver_id\"], value_type=ValueType.INT64)\n", + "source = SparkSource(\n", + " name=\"driver_stats_source\",\n", + " path=\"{dataset_uri}\",\n", + " file_format=\"parquet\",\n", + " timestamp_field=\"event_timestamp\",\n", + " created_timestamp_column=\"created\",\n", ")\n", - "parquet_buffer = io.BytesIO()\n", - "events.to_parquet(parquet_buffer, index=False)\n", - "s3.put_object(Bucket=s3_environment[\"S3_BUCKET\"], Key=S3_DATASET_KEY, Body=parquet_buffer.getvalue())\n", - "print({\"rows\": len(events), \"postgres_table\": POSTGRES_TABLE,\n", - " \"s3_uri\": f\"s3://{s3_environment['S3_BUCKET']}/{S3_DATASET_KEY}\"})\n", - "events.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Define and register offline features\n", - "\n", - "This uses `PostgreSQLSource` for point-in-time historical queries. `feast apply` writes definitions to the PostgreSQL-backed SQL registry and prepares the Redis online-store infrastructure. The Parquet snapshot remains in S3 for reproducibility and downstream dataset consumers; it is not used as a development-only `FileSource`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "(REPO / \"features.py\").write_text(textwrap.dedent(f\"\"\"\n", - " from datetime import timedelta\n", - " from feast import Entity, FeatureService, FeatureView, Field\n", - " from feast.infra.offline_stores.contrib.postgres_offline_store.postgres_source import PostgreSQLSource\n", - " from feast.types import Float32, Int64\n", - " from feast.value_type import ValueType\n", - "\n", - " driver = Entity(name=\"driver\", join_keys=[\"driver_id\"], value_type=ValueType.INT64)\n", - " driver_stats_source = PostgreSQLSource(\n", - " name=\"driver_stats_source\",\n", - " query=\"SELECT * FROM {POSTGRES_SCHEMA}.{POSTGRES_TABLE}\",\n", - " timestamp_field=\"event_timestamp\",\n", - " created_timestamp_column=\"created\",\n", - " )\n", - " driver_hourly_stats = FeatureView(\n", - " name=\"driver_hourly_stats\", entities=[driver], ttl=timedelta(days=365),\n", - " schema=[\n", - " Field(name=\"conv_rate\", dtype=Float32),\n", - " Field(name=\"acc_rate\", dtype=Float32),\n", - " Field(name=\"avg_daily_trips\", dtype=Int64),\n", - " ], online=True, source=driver_stats_source,\n", - " )\n", - " driver_activity_v1 = FeatureService(name=\"driver_activity_v1\", features=[driver_hourly_stats])\n", - "\"\"\"))\n", - "\n", - "# Copy the platform-generated config and make the online/registry certificates\n", - "# available to this notebook process. The serving pod will mount the original\n", - "# /tls paths; keep that copy separately for the model artifact.\n", - "runtime_config = client_config\n", - "local_config = runtime_config\n", - "for secret_name, original_path, local_name in [\n", - " (f\"feast-{FEATURESTORE_NAME}-offline-tls\", \"/tls/offline/tls.crt\", \"offline-tls.crt\"),\n", - " (f\"feast-{FEATURESTORE_NAME}-online-tls\", \"/tls/online/tls.crt\", \"online-tls.crt\"),\n", - " (f\"feast-{FEATURESTORE_NAME}-registry-tls\", \"/tls/registry/tls.crt\", \"registry-tls.crt\"),\n", - "]:\n", - " cert_b64 = kubectl(\"get\", \"secret\", secret_name, \"-n\", NAMESPACE, \"-o\", r\"jsonpath={.data.tls\\.crt}\", check=False)\n", - " if cert_b64:\n", - " cert_path = (REPO / local_name).resolve()\n", - " cert_path.write_bytes(base64.b64decode(cert_b64))\n", - " local_config = local_config.replace(original_path, str(cert_path))\n", - "(REPO / \"feature_store.yaml\").write_text(local_config)\n", - "(MODEL_DIR / \"feature_store.yaml\").write_text(runtime_config)\n", - "(MODEL_DIR / \"features.py\").write_text((REPO / \"features.py\").read_text())\n", - "subprocess.run([\"feast\", \"--chdir\", str(REPO), \"apply\"], check=True)\n", - "\n", - "print(\"Feature definitions registered in the SQL registry; Redis infrastructure prepared\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Retrieve offline features and train a sample model\n", - "\n", - "The model is deliberately a small NumPy linear classifier so the example does not require a second training image. The training matrix comes from Feast’s point-in-time PostgreSQL retrieval, not directly from the in-memory dataframe or S3 snapshot." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from feast import FeatureStore\n", + "view = FeatureView(\n", + " name=\"driver_hourly_stats\",\n", + " entities=[driver],\n", + " ttl=timedelta(days=365),\n", + " schema=[\n", + " Field(name=\"conv_rate\", dtype=Float32),\n", + " Field(name=\"acc_rate\", dtype=Float32),\n", + " Field(name=\"avg_daily_trips\", dtype=Int64),\n", + " ],\n", + " online=True,\n", + " source=source,\n", + ")\n", + "driver_activity_v1 = FeatureService(name=\"driver_activity_v1\", features=[view])\n", + "\"\"\")\n", "\n", - "store = FeatureStore(repo_path=str(REPO))\n", - "entity_df = events[[\"driver_id\", \"event_timestamp\", \"label\"]].copy()\n", + "subprocess.run([\"feast\", \"--chdir\", str(repo), \"apply\"], check=True)\n", + "store = FeatureStore(repo_path=str(repo))\n", + "entity_df = events.select(\"driver_id\", \"event_timestamp\", \"label\").toPandas()\n", "training_df = store.get_historical_features(\n", " entity_df=entity_df,\n", " features=[\n", @@ -310,23 +290,79 @@ " \"driver_hourly_stats:avg_daily_trips\",\n", " ],\n", ").to_df().dropna()\n", - "\n", - "feature_columns = [\"conv_rate\", \"acc_rate\", \"avg_daily_trips\"]\n", - "X = training_df[feature_columns].to_numpy(dtype=\"float64\")\n", + "columns = [\"conv_rate\", \"acc_rate\", \"avg_daily_trips\"]\n", + "x = training_df[columns].to_numpy(dtype=\"float64\")\n", "y = training_df[\"label\"].to_numpy(dtype=\"float64\")\n", - "X_bias = np.column_stack([np.ones(len(X)), X])\n", - "weights = np.linalg.pinv(X_bias) @ y\n", - "np.savez(MODEL_DIR / \"model.npz\", weights=weights, feature_columns=np.array(feature_columns))\n", - "print(\"historical rows:\", len(training_df), \"weights:\", weights)" + "weights = np.linalg.pinv(np.column_stack([np.ones(len(x)), x])) @ y\n", + "np.savez(\"/mnt/models/model.npz\", weights=weights, feature_columns=np.array(columns))\n", + "\n", + "store.materialize_incremental(events.agg(F.max(\"event_timestamp\")).first()[0] + pd.Timedelta(hours=1))\n", + "online = store.get_online_features(\n", + " features=store.get_feature_service(\"driver_activity_v1\"),\n", + " entity_rows=[{\"driver_id\": 1}, {\"driver_id\": 2}],\n", + ").to_df()\n", + "online.to_json(\"/mnt/models/online-sample.json\", orient=\"records\")\n", + "serving_config = copy.deepcopy(client_config)\n", + "serving_config.pop(\"offline_store\", None)\n", + "(model_repo / \"feature_store.yaml\").write_text(yaml.safe_dump(serving_config, sort_keys=False))\n", + "shutil.copy(\"/opt/feast-batch/server.py\", model_repo / \"server.py\")\n", + "print(json.dumps({\"rows\": len(training_df), \"dataset\": dataset_uri, \"online_rows\": len(online)}))\n", + "spark.stop()\n", + "'''\n", + "\n", + "server_source = r'''import os\n", + "import numpy as np\n", + "from fastapi import Body, FastAPI\n", + "from feast import FeatureStore\n", + "import uvicorn\n", + "\n", + "MODEL_NAME = os.getenv(\"MODEL_NAME\", \"feast-online-model\")\n", + "weights = np.load(\"/mnt/models/model.npz\")[\"weights\"]\n", + "store = FeatureStore(repo_path=\"/mnt/models/repo\")\n", + "feature_service = store.get_feature_service(\"driver_activity_v1\")\n", + "app = FastAPI()\n", + "\n", + "@app.get(\"/v2/health/live\")\n", + "@app.get(\"/v2/health/ready\")\n", + "def ready():\n", + " return {\"ready\": True}\n", + "\n", + "@app.get(\"/v2/models/{model_name}\")\n", + "@app.get(\"/v2/models/{model_name}/ready\")\n", + "def model_ready(model_name: str):\n", + " return {\"name\": model_name, \"ready\": model_name == MODEL_NAME}\n", + "\n", + "@app.post(\"/v2/models/{model_name}/infer\")\n", + "def infer(model_name: str, payload: dict = Body(...)):\n", + " ids = next(item for item in payload[\"inputs\"] if item[\"name\"] == \"driver_id\")[\"data\"]\n", + " rows = [{\"driver_id\": int(driver_id)} for driver_id in ids]\n", + " values = store.get_online_features(features=feature_service, entity_rows=rows).to_dict()\n", + " def column(name):\n", + " if name in values:\n", + " return values[name]\n", + " return values[next(key for key in values if key.endswith(\"__\" + name))]\n", + " x = np.column_stack([np.ones(len(ids)), column(\"conv_rate\"), column(\"acc_rate\"), column(\"avg_daily_trips\")])\n", + " prediction = (x @ weights).astype(\"float32\")\n", + " return {\"model_name\": model_name, \"outputs\": [{\"name\": \"prediction\", \"shape\": [len(ids)], \"datatype\": \"FP32\", \"data\": prediction.tolist()}]}\n", + "\n", + "if __name__ == \"__main__\":\n", + " uvicorn.run(app, host=\"0.0.0.0\", port=8080)\n", + "'''\n", + "\n", + "batch_dir = Path(\"feast-spark-batch\")\n", + "batch_dir.mkdir(exist_ok=True)\n", + "(batch_dir / \"batch.py\").write_text(batch_source)\n", + "(batch_dir / \"server.py\").write_text(server_source)\n", + "print(\"Prepared\", batch_dir)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 5. Materialize and verify online features\n", + "## 3. Submit the batch work as a SparkApplication\n", "\n", - "`materialize_incremental` copies the registered PostgreSQL features into Redis. The SQL registry cache may take up to its configured TTL to expose newly applied definitions to the online server, so the verification retries during that propagation window. The inference server below uses the same feature service and entity key at request time." + "The Spark Operator creates and monitors the driver and executor pods from this CR. A namespace-scoped service account gives the driver only the permissions it needs to manage its executors. The model PVC is mounted only in the driver; the dataset remains in S3.\n" ] }, { @@ -335,90 +371,169 @@ "metadata": {}, "outputs": [], "source": [ - "from feast.errors import FeatureViewNotFoundException\n", + "rbac_yaml = f\"\"\"\n", + "apiVersion: v1\n", + "kind: ServiceAccount\n", + "metadata:\n", + " name: {SPARK_SERVICE_ACCOUNT}\n", + " namespace: {NAMESPACE}\n", + "---\n", + "apiVersion: rbac.authorization.k8s.io/v1\n", + "kind: Role\n", + "metadata:\n", + " name: {SPARK_SERVICE_ACCOUNT}\n", + " namespace: {NAMESPACE}\n", + "rules:\n", + "- apiGroups: [\"\"]\n", + " resources: [\"pods\", \"pods/log\", \"services\", \"configmaps\"]\n", + " verbs: [\"get\", \"list\", \"watch\", \"create\", \"delete\", \"patch\"]\n", + "---\n", + "apiVersion: rbac.authorization.k8s.io/v1\n", + "kind: RoleBinding\n", + "metadata:\n", + " name: {SPARK_SERVICE_ACCOUNT}\n", + " namespace: {NAMESPACE}\n", + "subjects:\n", + "- kind: ServiceAccount\n", + " name: {SPARK_SERVICE_ACCOUNT}\n", + " namespace: {NAMESPACE}\n", + "roleRef:\n", + " apiGroup: rbac.authorization.k8s.io\n", + " kind: Role\n", + " name: {SPARK_SERVICE_ACCOUNT}\n", + "\"\"\"\n", + "kubectl(\"apply\", \"-f\", \"-\", input_text=rbac_yaml)\n", "\n", - "end_date = events[\"event_timestamp\"].max().to_pydatetime() + pd.Timedelta(hours=1)\n", - "store.materialize_incremental(end_date)\n", - "deadline = time.time() + 90\n", - "while True:\n", - " try:\n", - " online = store.get_online_features(\n", - " features=store.get_feature_service(\"driver_activity_v1\"),\n", - " entity_rows=[{\"driver_id\": 1}, {\"driver_id\": 2}],\n", - " ).to_df()\n", + "pvc_yaml = f\"\"\"\n", + "apiVersion: v1\n", + "kind: PersistentVolumeClaim\n", + "metadata:\n", + " name: {MODEL_PVC}\n", + " namespace: {NAMESPACE}\n", + "spec:\n", + " accessModes: [ReadWriteOnce]\n", + " resources:\n", + " requests:\n", + " storage: 1Gi\n", + "\"\"\"\n", + "kubectl(\"apply\", \"-f\", \"-\", input_text=pvc_yaml)\n", + "\n", + "batch_config_map = f\"{SPARK_APP}-code\"\n", + "config_map_yaml = kubectl(\n", + " \"create\", \"configmap\", batch_config_map, \"-n\", NAMESPACE,\n", + " f\"--from-file=batch.py={Path('feast-spark-batch/batch.py')}\",\n", + " f\"--from-file=server.py={Path('feast-spark-batch/server.py')}\",\n", + " \"--dry-run=client\", \"-o\", \"yaml\",\n", + ")\n", + "kubectl(\"apply\", \"-f\", \"-\", input_text=config_map_yaml)\n", + "\n", + "spark_application_yaml = f\"\"\"\n", + "apiVersion: sparkoperator.k8s.io/v1beta2\n", + "kind: SparkApplication\n", + "metadata:\n", + " name: {SPARK_APP}\n", + " namespace: {NAMESPACE}\n", + "spec:\n", + " type: Python\n", + " mode: cluster\n", + " image: {SPARK_IMAGE}\n", + " imagePullPolicy: IfNotPresent\n", + " mainApplicationFile: local:///opt/feast-batch/batch.py\n", + " sparkVersion: {SPARK_VERSION}\n", + " timeToLiveSeconds: 3600\n", + " restartPolicy:\n", + " type: Never\n", + " sparkConf:\n", + " spark.sql.session.timeZone: UTC\n", + " volumes:\n", + " - name: batch-code\n", + " configMap:\n", + " name: {batch_config_map}\n", + " - name: feast-client\n", + " configMap:\n", + " name: {client_config_map}\n", + " items:\n", + " - key: feature_store.yaml\n", + " path: feature_store.yaml\n", + " - name: model\n", + " persistentVolumeClaim:\n", + " claimName: {MODEL_PVC}\n", + " - name: online-tls\n", + " secret:\n", + " secretName: feast-{FEATURESTORE_NAME}-online-tls\n", + " - name: registry-tls\n", + " secret:\n", + " secretName: feast-{FEATURESTORE_NAME}-registry-tls\n", + " driver:\n", + " cores: 1\n", + " memory: 2g\n", + " serviceAccount: {SPARK_SERVICE_ACCOUNT}\n", + " env:\n", + " - name: FEAST_PROJECT\n", + " value: {FEAST_PROJECT}\n", + " - name: S3_DATASET_KEY\n", + " value: {S3_DATASET_KEY}\n", + " envFrom:\n", + " - secretRef:\n", + " name: {S3_CREDENTIALS_SECRET}\n", + " volumeMounts:\n", + " - name: batch-code\n", + " mountPath: /opt/feast-batch\n", + " readOnly: true\n", + " - name: feast-client\n", + " mountPath: /etc/feast\n", + " readOnly: true\n", + " - name: model\n", + " mountPath: /mnt/models\n", + " - name: online-tls\n", + " mountPath: /tls/online\n", + " readOnly: true\n", + " - name: registry-tls\n", + " mountPath: /tls/registry\n", + " readOnly: true\n", + " executor:\n", + " instances: 2\n", + " cores: 1\n", + " memory: 1g\n", + " envFrom:\n", + " - secretRef:\n", + " name: {S3_CREDENTIALS_SECRET}\n", + "\"\"\"\n", + "kubectl(\"delete\", \"sparkapplication\", SPARK_APP, \"-n\", NAMESPACE, \"--ignore-not-found\", \"--wait=true\")\n", + "kubectl(\"apply\", \"-f\", \"-\", input_text=spark_application_yaml)\n", + "\n", + "deadline = time.time() + 1200\n", + "terminal = {\"COMPLETED\", \"FAILED\", \"FAILED_SUBMISSION\", \"INVALIDATING\", \"UNKNOWN\"}\n", + "while time.time() < deadline:\n", + " state = kubectl(\n", + " \"get\", \"sparkapplication\", SPARK_APP, \"-n\", NAMESPACE,\n", + " \"-o\", \"jsonpath={.status.applicationState.state}\", check=False,\n", + " )\n", + " print(state or \"SUBMITTED\")\n", + " if state in terminal:\n", " break\n", - " except FeatureViewNotFoundException:\n", - " if time.time() >= deadline:\n", - " raise\n", - " time.sleep(5)\n", - "online" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. Define the online model server\n", + " time.sleep(10)\n", + "else:\n", + " raise TimeoutError(\"SparkApplication did not reach a terminal state\")\n", "\n", - "The server loads the trained NumPy weights from the model PVC, queries Feast online features, and returns a KServe v2 response. The Feast online and registry TLS secrets are mounted by the `ServingRuntime`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_source = textwrap.dedent(\"\"\"\n", - " import os\n", - " import numpy as np\n", - " from fastapi import Body, FastAPI\n", - " from feast import FeatureStore\n", - " import uvicorn\n", - "\n", - " MODEL_NAME = os.getenv(\"MODEL_NAME\", \"feast-online-model\")\n", - " weights = np.load(\"/mnt/models/model.npz\")[\"weights\"]\n", - " store = FeatureStore(repo_path=\"/mnt/models\")\n", - " feature_service = store.get_feature_service(\"driver_activity_v1\")\n", - " app = FastAPI()\n", - "\n", - " @app.get(\"/v2/health/live\")\n", - " @app.get(\"/v2/health/ready\")\n", - " def ready():\n", - " return {\"ready\": True}\n", - "\n", - " @app.get(\"/v2/models/{model_name}\")\n", - " @app.get(\"/v2/models/{model_name}/ready\")\n", - " def model_ready(model_name: str):\n", - " return {\"name\": model_name, \"ready\": model_name == MODEL_NAME}\n", - "\n", - " @app.post(\"/v2/models/{model_name}/infer\")\n", - " def infer(model_name: str, payload: dict = Body(...)):\n", - " ids = next(item for item in payload[\"inputs\"] if item[\"name\"] == \"driver_id\")[\"data\"]\n", - " rows = [{\"driver_id\": int(driver_id)} for driver_id in ids]\n", - " values = store.get_online_features(features=feature_service, entity_rows=rows).to_dict()\n", - " def column(name):\n", - " if name in values:\n", - " return values[name]\n", - " return values[next(key for key in values if key.endswith(\"__\" + name))]\n", - " X = np.column_stack([np.ones(len(ids)), column(\"conv_rate\"), column(\"acc_rate\"), column(\"avg_daily_trips\")])\n", - " prediction = (X @ weights).astype(\"float32\")\n", - " return {\"model_name\": model_name, \"outputs\": [{\"name\": \"prediction\", \"shape\": [len(ids)], \"datatype\": \"FP32\", \"data\": prediction.tolist()}]}\n", - "\n", - " if __name__ == \"__main__\":\n", - " uvicorn.run(app, host=\"0.0.0.0\", port=8080)\n", - "\"\"\")\n", - "(MODEL_DIR / \"server.py\").write_text(server_source)\n", - "print(MODEL_DIR / \"server.py\")" + "if state != \"COMPLETED\":\n", + " driver_pod = kubectl(\n", + " \"get\", \"sparkapplication\", SPARK_APP, \"-n\", NAMESPACE,\n", + " \"-o\", \"jsonpath={.status.driverInfo.podName}\", check=False,\n", + " )\n", + " logs = kubectl(\"logs\", driver_pod, \"-n\", NAMESPACE, \"--tail=300\", check=False) if driver_pod else \"\"\n", + " raise RuntimeError(f\"SparkApplication ended in {state}\\n{logs}\")\n", + "\n", + "print(\"Spark batch completed\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 7. Stage the model and start KServe\n", + "## 4. Inspect the offline-to-online result\n", "\n", - "Create a model PVC before running this section. The temporary stager pod copies the local model artifact into it; KServe then consumes it through `storageUri: pvc://...`. The runtime mounts the Feast online and registry certificates at the paths referenced by the generated client configuration. The `RawDeployment` annotation allows a direct predictor Service where the cluster permits it; the final cell verifies the predictor Deployment itself so it also works on clusters whose KServe policy selects Standard mode." + "The driver writes a small verification sample to the model PVC after materialization. Inspecting it through a short-lived pod confirms that the Spark batch completed the S3 historical path and that Feast could read the materialized Redis values.\n" ] }, { @@ -427,31 +542,18 @@ "metadata": {}, "outputs": [], "source": [ - "pvc_yaml = f\"\"\"\n", - "apiVersion: v1\n", - "kind: PersistentVolumeClaim\n", - "metadata:\n", - " name: {MODEL_PVC}\n", - " namespace: {NAMESPACE}\n", - "spec:\n", - " accessModes: [ReadWriteOnce]\n", - " resources:\n", - " requests:\n", - " storage: 1Gi\n", - "\"\"\"\n", - "kubectl(\"apply\", \"-f\", \"-\", input_text=pvc_yaml)\n", - "stager = f\"\"\"\n", + "inspector_yaml = f\"\"\"\n", "apiVersion: v1\n", "kind: Pod\n", "metadata:\n", - " name: feast-model-stager\n", + " name: feast-model-inspector\n", " namespace: {NAMESPACE}\n", "spec:\n", " restartPolicy: Never\n", " containers:\n", - " - name: stager\n", + " - name: inspector\n", " image: {MODEL_IMAGE}\n", - " command: [bash, -c, sleep 3600]\n", + " command: [bash, -c, cat /mnt/models/online-sample.json]\n", " volumeMounts:\n", " - name: model\n", " mountPath: /mnt/models\n", @@ -460,14 +562,28 @@ " persistentVolumeClaim:\n", " claimName: {MODEL_PVC}\n", "\"\"\"\n", - "kubectl(\"apply\", \"-f\", \"-\", input_text=stager)\n", - "kubectl(\"wait\", \"--for=condition=Ready\", \"pod/feast-model-stager\", \"-n\", NAMESPACE, \"--timeout=180s\")\n", - "kubectl(\"cp\", str(MODEL_DIR / \"model.npz\"), f\"{NAMESPACE}/feast-model-stager:/mnt/models/model.npz\")\n", - "kubectl(\"cp\", str(MODEL_DIR / \"server.py\"), f\"{NAMESPACE}/feast-model-stager:/mnt/models/server.py\")\n", - "kubectl(\"cp\", str(MODEL_DIR / \"features.py\"), f\"{NAMESPACE}/feast-model-stager:/mnt/models/features.py\")\n", - "kubectl(\"cp\", str(MODEL_DIR / \"feature_store.yaml\"), f\"{NAMESPACE}/feast-model-stager:/mnt/models/feature_store.yaml\")\n", - "kubectl(\"delete\", \"pod\", \"feast-model-stager\", \"-n\", NAMESPACE, \"--wait=true\")\n", + "kubectl(\"delete\", \"pod\", \"feast-model-inspector\", \"-n\", NAMESPACE, \"--ignore-not-found\", \"--wait=true\")\n", + "kubectl(\"apply\", \"-f\", \"-\", input_text=inspector_yaml)\n", + "kubectl(\"wait\", \"--for=jsonpath={.status.phase}=Succeeded\", \"pod/feast-model-inspector\", \"-n\", NAMESPACE, \"--timeout=180s\")\n", + "print(kubectl(\"logs\", \"feast-model-inspector\", \"-n\", NAMESPACE))\n", + "kubectl(\"delete\", \"pod\", \"feast-model-inspector\", \"-n\", NAMESPACE, \"--wait=true\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Deploy the online model with KServe\n", "\n", + "The model server loads the weights and Feast definitions from the PVC, queries Redis through the operator-managed Feast online service, and exposes KServe's v2 inference protocol. The online and registry TLS Secrets are mounted at the paths in the generated Feast client configuration.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "runtime_yaml = f\"\"\"\n", "apiVersion: serving.kserve.io/v1alpha1\n", "kind: ServingRuntime\n", @@ -478,7 +594,7 @@ " containers:\n", " - name: kserve-container\n", " image: {MODEL_IMAGE}\n", - " command: [python, /mnt/models/server.py]\n", + " command: [python, /mnt/models/repo/server.py]\n", " ports:\n", " - containerPort: 8080\n", " name: http1\n", @@ -532,16 +648,16 @@ "\"\"\"\n", "kubectl(\"apply\", \"-f\", \"-\", input_text=runtime_yaml)\n", "kubectl(\"apply\", \"-f\", \"-\", input_text=isvc_yaml)\n", - "print(kubectl(\"get\", \"inferenceservice\", MODEL_NAME, \"-n\", NAMESPACE))" + "print(kubectl(\"get\", \"inferenceservice\", MODEL_NAME, \"-n\", NAMESPACE))\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 8. Wait for the service and send an online-feature prediction\n", + "## 6. Send an online-feature prediction\n", "\n", - "When the predictor Deployment has an available replica, send entity IDs to the KServe v2 endpoint. The server looks those IDs up in Feast’s online store and combines the returned features with the trained weights." + "After the predictor has an available replica, send driver IDs to the KServe v2 endpoint. The server looks up their materialized features in Redis and combines those values with the model trained by the `SparkApplication`.\n" ] }, { @@ -553,7 +669,10 @@ "deadline = time.time() + 900\n", "predictor_deployment = f\"{MODEL_NAME}-predictor\"\n", "while time.time() < deadline:\n", - " deployment = json.loads(kubectl(\"get\", \"deployment\", predictor_deployment, \"-n\", NAMESPACE, \"-o\", \"json\"))\n", + " deployment_text = kubectl(\n", + " \"get\", \"deployment\", predictor_deployment, \"-n\", NAMESPACE, \"-o\", \"json\", check=False\n", + " )\n", + " deployment = json.loads(deployment_text) if deployment_text else {}\n", " available = deployment.get(\"status\", {}).get(\"availableReplicas\", 0) or 0\n", " print({\"availableReplicas\": available})\n", " if available >= 1:\n", @@ -562,12 +681,10 @@ "else:\n", " raise TimeoutError(\"KServe predictor deployment did not become available\")\n", "\n", - "isvc_status = json.loads(kubectl(\"get\", \"inferenceservice\", MODEL_NAME, \"-n\", NAMESPACE, \"-o\", \"json\"))\n", - "print(\"Ingress URL (if configured):\", isvc_status.get(\"status\", {}).get(\"url\"))\n", - "\n", "port_forward = subprocess.Popen(\n", " [\"kubectl\", \"port-forward\", f\"service/{predictor_deployment}\", \"18080:80\", \"-n\", NAMESPACE],\n", - " stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,\n", + " stdout=subprocess.DEVNULL,\n", + " stderr=subprocess.DEVNULL,\n", ")\n", "base_url = \"http://127.0.0.1:18080\"\n", "try:\n", @@ -590,21 +707,27 @@ " timeout=30,\n", " )\n", " response.raise_for_status()\n", - " prediction = response.json()\n", - " print(json.dumps(prediction, indent=2))\n", + " print(json.dumps(response.json(), indent=2))\n", "finally:\n", " port_forward.terminate()\n", " try:\n", " port_forward.wait(timeout=5)\n", " except subprocess.TimeoutExpired:\n", " port_forward.kill()\n", - " port_forward.wait()" + " port_forward.wait()\n" ] } ], "metadata": { - "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, - "language_info": {"name": "python", "version": "3.11"} + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + } }, "nbformat": 4, "nbformat_minor": 5 diff --git a/e2e/cases/c16_feast_offline_online.sh b/e2e/cases/c16_feast_offline_online.sh index 182a7c11..6ecb4b01 100755 --- a/e2e/cases/c16_feast_offline_online.sh +++ b/e2e/cases/c16_feast_offline_online.sh @@ -1,9 +1,13 @@ #!/usr/bin/env bash -# C16: Feast Parquet -> historical features -> NumPy model -> online KServe prediction. +# C16: SparkApplication + S3 Parquet -> Feast historical features -> Redis -> KServe. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" source "$HERE/../lib.sh" + require_env FEAST_NAMESPACE "namespace for Feast e2e resources" +require_env FEAST_IMAGE "Feast model-server image from the cluster registry" +require_env FEAST_SPARK_IMAGE "Spark runtime with Feast Spark and Hadoop S3A support" + NS="$FEAST_NAMESPACE" PROJECT="feast_demo" RUN_ID="$(printf '%05x' $$)-$(date -u +%s)" @@ -11,15 +15,21 @@ FS_NAME="feast-e2e-$RUN_ID" MODEL_PVC="$FS_NAME-model" RUNTIME="$FS_NAME-runtime" ISVC="$FS_NAME-isvc" -JOB="$FS_NAME-job" -CM="$FS_NAME-runner" -IMAGE="$FEAST_IMAGE"; [ -n "$IMAGE" ] || IMAGE=build-harbor.alauda.cn/mlops/feast/feature-server:0.61.0 +SPARK_APP="$FS_NAME-spark" +SPARK_SA="$FS_NAME-spark" +CM="$FS_NAME-code" +S3_KEY="e2e/$FS_NAME/driver_stats" TMP="$(mktemp -d)" PF="" + cleanup() { [ -n "$PF" ] && kill "$PF" 2>/dev/null || true if [ "$FEAST_KEEP_RESOURCES" != 1 ]; then - for item in "inferenceservice $ISVC" "servingruntime $RUNTIME" "job $JOB" "configmap $CM" "pvc $MODEL_PVC" "featurestore $FS_NAME"; do + for item in \ + "inferenceservice $ISVC" "servingruntime $RUNTIME" \ + "sparkapplication $SPARK_APP" "configmap $CM" "pvc $MODEL_PVC" \ + "featurestore $FS_NAME" "rolebinding $SPARK_SA" "role $SPARK_SA" \ + "serviceaccount $SPARK_SA"; do set -- $item feast_kc -n "$NS" delete "$1" "$2" --ignore-not-found --wait=false >/dev/null 2>&1 || true done @@ -27,10 +37,29 @@ cleanup() { rm -rf "$TMP" } trap cleanup EXIT -feast_kc get crd featurestores.feast.dev >/dev/null 2>&1 || { log "Feast CRD missing; skipping"; exit "$E2E_SKIP_RC"; } -feast_kc get crd inferenceservices.serving.kserve.io >/dev/null 2>&1 || { log "KServe CRD missing; skipping"; exit "$E2E_SKIP_RC"; } + +feast_kc get crd featurestores.feast.dev >/dev/null 2>&1 || { + log "Feast CRD missing; skipping" + exit "$E2E_SKIP_RC" +} +feast_kc get crd sparkapplications.sparkoperator.k8s.io >/dev/null 2>&1 || { + log "OLM Spark Operator CRD missing; skipping" + exit "$E2E_SKIP_RC" +} +feast_kc get crd inferenceservices.serving.kserve.io >/dev/null 2>&1 || { + log "KServe CRD missing; skipping" + exit "$E2E_SKIP_RC" +} + feast_kc create namespace "$NS" --dry-run=client -o yaml | feast_kc apply -f - >/dev/null feast_kc create namespace feast-operator-system --dry-run=client -o yaml | feast_kc apply -f - >/dev/null +for secret in "$FEAST_DATA_STORES_SECRET" "$FEAST_S3_CREDENTIALS_SECRET"; do + feast_kc -n "$NS" get secret "$secret" >/dev/null 2>&1 || { + log "required Secret $NS/$secret missing; skipping" + exit "$E2E_SKIP_RC" + } +done + cat </dev/null || true)" [ "$phase" = Ready ] && break - [ "$phase" = Failed ] && { feast_kc -n "$NS" get featurestore "$FS_NAME" -o yaml >&2; exit 1; } + [ "$phase" = Failed ] && { + feast_kc -n "$NS" get featurestore "$FS_NAME" -o yaml >&2 + exit 1 + } sleep 10 done [ "$phase" = Ready ] || { log "FeatureStore did not become Ready"; exit 1; } + CLIENT="$(feast_kc -n "$NS" get featurestore "$FS_NAME" -o jsonpath='{.status.clientConfigMap}')" ONLINE_TLS="feast-$FS_NAME-online-tls" REGISTRY_TLS="feast-$FS_NAME-registry-tls" -cat >"$TMP/features.py" <<'PY' -from datetime import timedelta -from feast import Entity, FeatureService, FeatureView, Field, FileSource -from feast.data_format import ParquetFormat +cat >"$TMP/batch.py" <<'PY' +import copy +import json +import os +import shutil +import subprocess +from pathlib import Path +from urllib.parse import urlparse + +import numpy as np +import pandas as pd +import yaml +from feast import FeatureStore +from pyspark.sql import SparkSession, functions as F + +project = os.environ["FEAST_PROJECT"] +bucket = os.environ["S3_BUCKET"] +dataset_key = os.environ["S3_DATASET_KEY"].strip("/") +dataset_uri = f"s3a://{bucket}/{dataset_key}" +region = os.environ["AWS_DEFAULT_REGION"] +repo = Path("/tmp/feast-repo") +model_repo = Path("/mnt/models/repo") +repo.mkdir(parents=True, exist_ok=True) +model_repo.mkdir(parents=True, exist_ok=True) + +spark = SparkSession.builder.appName("feast-offline-online-e2e").getOrCreate() +endpoint = urlparse(os.environ["S3_ENDPOINT_URL"]) +hadoop = spark.sparkContext._jsc.hadoopConfiguration() +hadoop.set("fs.s3a.endpoint", endpoint.netloc or endpoint.path) +hadoop.set("fs.s3a.endpoint.region", region) +hadoop.set("fs.s3a.path.style.access", "true") +hadoop.set("fs.s3a.connection.ssl.enabled", str(endpoint.scheme == "https").lower()) +hadoop.set("fs.s3a.access.key", os.environ["AWS_ACCESS_KEY_ID"]) +hadoop.set("fs.s3a.secret.key", os.environ["AWS_SECRET_ACCESS_KEY"]) + +events = ( + spark.range(240) + .withColumn("driver_id", (F.col("id") % 12 + 1).cast("long")) + .withColumn("event_timestamp", F.timestamp_seconds(F.lit(1767225600) + F.col("id") * 3600)) + .withColumn("created", F.col("event_timestamp") + F.expr("INTERVAL 1 MINUTE")) + .withColumn("conv_rate", (F.lit(0.25) + F.lit(0.55) * F.rand(7)).cast("float")) + .withColumn("acc_rate", (F.lit(0.50) + F.lit(0.45) * F.rand(11)).cast("float")) + .withColumn("avg_daily_trips", F.floor(F.lit(2) + F.lit(18) * F.rand(13)).cast("long")) + .withColumn( + "label", + ((F.col("conv_rate") * 2 + F.col("acc_rate") + F.col("avg_daily_trips") / 20) > 1.8).cast("long"), + ) + .drop("id") +) +events.repartition(4, "driver_id").write.mode("overwrite").partitionBy("driver_id").parquet(dataset_uri) + +client_config = yaml.safe_load(Path("/etc/feast/feature_store.yaml").read_text()) +batch_config = copy.deepcopy(client_config) +batch_config["offline_store"] = { + "type": "spark", + "spark_conf": { + "spark.sql.session.timeZone": "UTC", + "spark.hadoop.fs.s3a.endpoint": endpoint.netloc or endpoint.path, + "spark.hadoop.fs.s3a.endpoint.region": region, + "spark.hadoop.fs.s3a.path.style.access": "true", + "spark.hadoop.fs.s3a.connection.ssl.enabled": str(endpoint.scheme == "https").lower(), + }, +} +(repo / "feature_store.yaml").write_text(yaml.safe_dump(batch_config, sort_keys=False)) +(repo / "features.py").write_text(f'''from datetime import timedelta +from feast import Entity, FeatureService, FeatureView, Field +from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import SparkSource from feast.types import Float32, Int64 from feast.value_type import ValueType driver = Entity(name="driver", join_keys=["driver_id"], value_type=ValueType.INT64) -source = FileSource(name="driver_stats_source", path="data/driver_stats.parquet", - file_format=ParquetFormat(), timestamp_field="event_timestamp", - created_timestamp_column="created") -view = FeatureView(name="driver_hourly_stats", entities=[driver], ttl=timedelta(days=365), - schema=[Field(name="conv_rate", dtype=Float32), Field(name="acc_rate", dtype=Float32), - Field(name="avg_daily_trips", dtype=Int64)], online=True, source=source) +source = SparkSource( + name="driver_stats_source", + path="{dataset_uri}", + file_format="parquet", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) +view = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=365), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=source, +) driver_activity_v1 = FeatureService(name="driver_activity_v1", features=[view]) -PY +''') -# The remote online-store client intentionally has a no-op infrastructure update -# in Feast 0.61. Run apply once in the operand's local repository so its SQLite -# table exists before a remote materialize call writes rows to the online server. -online_pod="" -for _ in $(seq 1 60); do - online_pod="$(feast_kc -n "$NS" get pods -l "feast.dev/name=$FS_NAME" \ - -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" - [ -n "$online_pod" ] && break - sleep 5 -done -[ -n "$online_pod" ] || { log "Feast online pod did not appear"; exit 1; } -feast_kc -n "$NS" wait --for=condition=Ready "pod/$online_pod" --timeout=300s >/dev/null -feast_kc -n "$NS" cp "$TMP/features.py" \ - "$NS/$online_pod:/feast-data/$PROJECT/feature_repo/feature_definitions.py" -c online -feast_kc -n "$NS" exec "$online_pod" -c online -- \ - bash -c "cd /feast-data/$PROJECT/feature_repo && feast apply" - -cat >"$TMP/run.sh" <<'RUN' -#!/usr/bin/env bash -set -euo pipefail -R=/mnt/models/repo -mkdir -p "$R/data" -cp /etc/feast/feature_store.yaml "$R/feature_store.yaml" -cp /runner/features.py "$R/features.py" -python - <<'PY' -import numpy as np, pandas as pd -rng=np.random.default_rng(7); n=240 -df=pd.DataFrame({"driver_id":(np.arange(n)%12+1).astype("int64"), - "event_timestamp":pd.date_range("2026-01-01",periods=n,freq="h",tz="UTC")}) -df["created"]=df.event_timestamp+pd.to_timedelta(1,unit="m") -df["conv_rate"]=(.25+.55*rng.random(n)).astype("float32") -df["acc_rate"]=(.50+.45*rng.random(n)).astype("float32") -df["avg_daily_trips"]=rng.integers(2,20,size=n).astype("int64") -df["label"]=((df.conv_rate*2+df.acc_rate+df.avg_daily_trips/20)>1.8).astype("int64") -df.to_parquet("/mnt/models/repo/data/driver_stats.parquet",index=False) -PY -feast --chdir "$R" apply -python - <<'PY' -import numpy as np, pandas as pd -from feast import FeatureStore -r="/mnt/models/repo"; s=FeatureStore(repo_path=r); raw=pd.read_parquet(r+"/data/driver_stats.parquet") -t=s.get_historical_features(entity_df=raw[["driver_id","event_timestamp","label"]], - features=["driver_hourly_stats:conv_rate","driver_hourly_stats:acc_rate", - "driver_hourly_stats:avg_daily_trips"]).to_df().dropna() -x=t[["conv_rate","acc_rate","avg_daily_trips"]].to_numpy(float); y=t.label.to_numpy(float) -w=np.linalg.pinv(np.column_stack([np.ones(len(x)),x]))@y -np.savez("/mnt/models/model.npz",weights=w) -s.materialize_incremental(raw.event_timestamp.max().to_pydatetime()+pd.Timedelta(hours=1)) -print("historical_rows",len(t),"weights",w.tolist()) +subprocess.run(["feast", "--chdir", str(repo), "apply"], check=True) +store = FeatureStore(repo_path=str(repo)) +entity_df = events.select("driver_id", "event_timestamp", "label").toPandas() +training = store.get_historical_features( + entity_df=entity_df, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + ], +).to_df().dropna() +columns = ["conv_rate", "acc_rate", "avg_daily_trips"] +x = training[columns].to_numpy(dtype="float64") +y = training["label"].to_numpy(dtype="float64") +weights = np.linalg.pinv(np.column_stack([np.ones(len(x)), x])) @ y +np.savez("/mnt/models/model.npz", weights=weights, feature_columns=np.array(columns)) + +end_date = events.agg(F.max("event_timestamp")).first()[0] + pd.Timedelta(hours=1) +store.materialize_incremental(end_date) +online = store.get_online_features( + features=store.get_feature_service("driver_activity_v1"), + entity_rows=[{"driver_id": 1}, {"driver_id": 2}], +).to_df() +if len(online) != 2 or online[columns].isna().any().any(): + raise RuntimeError(f"online feature verification failed: {online}") + +serving_config = copy.deepcopy(client_config) +serving_config.pop("offline_store", None) +(model_repo / "feature_store.yaml").write_text(yaml.safe_dump(serving_config, sort_keys=False)) +shutil.copy("/opt/feast-batch/server.py", model_repo / "server.py") +print(json.dumps({"historical_rows": len(training), "online_rows": len(online)})) + +if os.environ.get("CLEANUP_S3") == "true": + jvm = spark.sparkContext._jvm + filesystem = jvm.org.apache.hadoop.fs.FileSystem.get(jvm.java.net.URI.create(dataset_uri), hadoop) + filesystem.delete(jvm.org.apache.hadoop.fs.Path(dataset_uri), True) +spark.stop() PY -cp "$R/feature_store.yaml" /mnt/models/feature_store.yaml -cp "$R/features.py" /mnt/models/features.py -cp /runner/server.py /mnt/models/server.py -RUN + cat >"$TMP/server.py" <<'PY' -import os, numpy as np, uvicorn +import os +import numpy as np +import uvicorn from fastapi import Body, FastAPI from feast import FeatureStore -name=os.getenv("MODEL_NAME","feast-online-model") -w=np.load("/mnt/models/model.npz")["weights"]; s=FeatureStore(repo_path="/mnt/models") -fs=s.get_feature_service("driver_activity_v1"); app=FastAPI() + +name = os.getenv("MODEL_NAME", "feast-online-model") +weights = np.load("/mnt/models/model.npz")["weights"] +store = FeatureStore(repo_path="/mnt/models/repo") +service = store.get_feature_service("driver_activity_v1") +app = FastAPI() + @app.get("/v2/health/ready") @app.get("/v2/health/live") -def health(): return {"ready":True} +def health(): + return {"ready": True} + @app.get("/v2/models/{model_name}") @app.get("/v2/models/{model_name}/ready") -def ready(model_name): return {"name":model_name,"ready":model_name==name} +def ready(model_name): + return {"name": model_name, "ready": model_name == name} + @app.post("/v2/models/{model_name}/infer") def infer(model_name, payload: dict = Body(...)): - ids=next(x for x in payload["inputs"] if x["name"]=="driver_id")["data"] - values=s.get_online_features(features=fs,entity_rows=[{"driver_id":int(x)} for x in ids]).to_dict() - def col(n): - return values[n] if n in values else values[next(k for k in values if k.endswith("__"+n))] - x=np.column_stack([np.ones(len(ids)),col("conv_rate"),col("acc_rate"),col("avg_daily_trips")]) - return {"model_name":model_name,"outputs":[{"name":"prediction","shape":[len(ids)],"datatype":"FP32","data":(x@w).astype("float32").tolist()}]} -if __name__=="__main__": uvicorn.run(app,host="0.0.0.0",port=8080) + ids = next(item for item in payload["inputs"] if item["name"] == "driver_id")["data"] + values = store.get_online_features( + features=service, + entity_rows=[{"driver_id": int(value)} for value in ids], + ).to_dict() + def column(column_name): + if column_name in values: + return values[column_name] + return values[next(key for key in values if key.endswith("__" + column_name))] + matrix = np.column_stack([ + np.ones(len(ids)), column("conv_rate"), column("acc_rate"), column("avg_daily_trips") + ]) + predictions = (matrix @ weights).astype("float32").tolist() + return { + "model_name": model_name, + "outputs": [{"name": "prediction", "shape": [len(ids)], "datatype": "FP32", "data": predictions}], + } + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8080) PY +feast_kc -n "$NS" create configmap "$CM" \ + --from-file=batch.py="$TMP/batch.py" --from-file=server.py="$TMP/server.py" \ + --dry-run=client -o yaml | feast_kc apply -f - >/dev/null + cat </dev/null cat </dev/null || true)" + case "$state" in + COMPLETED|FAILED|FAILED_SUBMISSION|INVALIDATING|UNKNOWN) break ;; + esac + sleep 10 +done +driver="$(feast_kc -n "$NS" get sparkapplication "$SPARK_APP" \ + -o jsonpath='{.status.driverInfo.podName}' 2>/dev/null || true)" +[ "$state" = COMPLETED ] || { + log "SparkApplication ended in state ${state:-unset}" + [ -n "$driver" ] && feast_kc -n "$NS" logs "$driver" --tail=300 >&2 || true + exit 1 +} +[ -n "$driver" ] && feast_kc -n "$NS" logs "$driver" --tail=100 || true cat </dev/null || true)" @@ -241,11 +424,16 @@ done feast_kc -n "$NS" get inferenceservice "$ISVC" -o yaml >&2 || true exit 1 } -SVC="$ISVC-predictor" -feast_kc -n "$NS" get service "$SVC" >/dev/null -feast_kc -n "$NS" port-forward "service/$SVC" 18080:80 >"$TMP/pf.log" 2>&1 & + +feast_kc -n "$NS" port-forward "service/$PREDICTOR_DEPLOY" 18080:80 >"$TMP/pf.log" 2>&1 & PF=$! -for _ in $(seq 1 30); do curl -fsS http://127.0.0.1:18080/v2/health/ready >/dev/null 2>&1 && break; sleep 2; done -response="$(curl -fsS -X POST "http://127.0.0.1:18080/v2/models/$ISVC/infer" -H 'Content-Type: application/json' -d '{"inputs":[{"name":"driver_id","shape":[2],"datatype":"INT64","data":[1,2]}]}')" -echo "$response"; echo "$response" | grep -q prediction -log "C16: Feast offline-to-online inference demo passed" +for _ in $(seq 1 30); do + curl -fsS http://127.0.0.1:18080/v2/health/ready >/dev/null 2>&1 && break + sleep 2 +done +response="$(curl -fsS -X POST "http://127.0.0.1:18080/v2/models/$ISVC/infer" \ + -H 'Content-Type: application/json' \ + -d '{"inputs":[{"name":"driver_id","shape":[2],"datatype":"INT64","data":[1,2]}]}')" +echo "$response" +echo "$response" | grep -q prediction +log "C16: SparkApplication S3 offline-to-Redis online inference passed" diff --git a/e2e/lib.sh b/e2e/lib.sh index b406c99f..dc5cd3f5 100644 --- a/e2e/lib.sh +++ b/e2e/lib.sh @@ -19,6 +19,8 @@ case "${E2E_SKIP_RC}" in esac # Required per case: GPU_NAMESPACE, NPU_NAMESPACE, or FEAST_NAMESPACE. +# Feast C16 also requires FEAST_IMAGE and FEAST_SPARK_IMAGE; its durable-store +# Secret names default to feast-data-stores and feast-s3-credentials. # Optional kube target: GPU_CONTEXT/GPU_KUBECONFIG/NPU_CONTEXT/NPU_KUBECONFIG. # Optional Docker Hub mirrors: GPU_DH_MIRROR/NPU_DH_MIRROR. # Optional private registry access: E2E_IMAGE_PULL_SECRET. @@ -35,6 +37,10 @@ FEAST_CONTEXT="${FEAST_CONTEXT:-}" FEAST_KUBECONFIG="${FEAST_KUBECONFIG:-}" FEAST_NAMESPACE="${FEAST_NAMESPACE:-}" FEAST_IMAGE="${FEAST_IMAGE:-}" +FEAST_SPARK_IMAGE="${FEAST_SPARK_IMAGE:-}" +FEAST_SPARK_VERSION="${FEAST_SPARK_VERSION:-4.0.1}" +FEAST_DATA_STORES_SECRET="${FEAST_DATA_STORES_SECRET:-feast-data-stores}" +FEAST_S3_CREDENTIALS_SECRET="${FEAST_S3_CREDENTIALS_SECRET:-feast-s3-credentials}" FEAST_STORAGE_CLASS="${FEAST_STORAGE_CLASS:-}" FEAST_KEEP_RESOURCES="${FEAST_KEEP_RESOURCES:-0}" From 1259bd53a4015c913ac60b34b655ff14b493cac1 Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Thu, 20 Aug 2026 11:03:21 +0800 Subject: [PATCH 5/7] docs: configure Spark S3A through public API --- .../feast-offline-to-online-inference.ipynb | 25 +++++++++------- e2e/cases/c16_feast_offline_online.sh | 29 +++++++++---------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/docs/en/train/guides/feast-offline-to-online-inference.ipynb b/docs/en/train/guides/feast-offline-to-online-inference.ipynb index 2d89f69e..fa1abc92 100644 --- a/docs/en/train/guides/feast-offline-to-online-inference.ipynb +++ b/docs/en/train/guides/feast-offline-to-online-inference.ipynb @@ -177,7 +177,7 @@ "\n", "This section creates the code that the Spark Operator will mount into the driver. The application generates deterministic synthetic driver events with Spark, writes partitioned Parquet to the configured S3-compatible service, registers a Feast `SparkSource`, performs the point-in-time historical join, trains the sample model, and materializes the same features into Redis.\n", "\n", - "The S3 endpoint and credentials come from a Kubernetes Secret. The code enables path-style access so it works with SeaweedFS and similar S3-compatible services; use your service's TLS endpoint in production.\n" + "The S3 endpoint and credentials come from a Kubernetes Secret. The code configures S3A through the public `SparkSession.builder.config()` API before the Spark context is created, while S3A reads the AWS credentials from the driver and executor environments. It enables path-style access so it works with SeaweedFS and similar S3-compatible services; use your service's TLS endpoint in production.\n" ] }, { @@ -210,15 +210,18 @@ "repo.mkdir(parents=True, exist_ok=True)\n", "model_repo.mkdir(parents=True, exist_ok=True)\n", "\n", - "spark = SparkSession.builder.appName(\"feast-offline-batch\").getOrCreate()\n", "endpoint = urlparse(os.environ[\"S3_ENDPOINT_URL\"])\n", - "hadoop = spark.sparkContext._jsc.hadoopConfiguration()\n", - "hadoop.set(\"fs.s3a.endpoint\", endpoint.netloc or endpoint.path)\n", - "hadoop.set(\"fs.s3a.endpoint.region\", region)\n", - "hadoop.set(\"fs.s3a.path.style.access\", \"true\")\n", - "hadoop.set(\"fs.s3a.connection.ssl.enabled\", str(endpoint.scheme == \"https\").lower())\n", - "hadoop.set(\"fs.s3a.access.key\", os.environ[\"AWS_ACCESS_KEY_ID\"])\n", - "hadoop.set(\"fs.s3a.secret.key\", os.environ[\"AWS_SECRET_ACCESS_KEY\"])\n", + "endpoint_host = endpoint.netloc or endpoint.path\n", + "ssl_enabled = str(endpoint.scheme == \"https\").lower()\n", + "spark = (\n", + " SparkSession.builder\n", + " .appName(\"feast-offline-batch\")\n", + " .config(\"spark.hadoop.fs.s3a.endpoint\", endpoint_host)\n", + " .config(\"spark.hadoop.fs.s3a.endpoint.region\", region)\n", + " .config(\"spark.hadoop.fs.s3a.path.style.access\", \"true\")\n", + " .config(\"spark.hadoop.fs.s3a.connection.ssl.enabled\", ssl_enabled)\n", + " .getOrCreate()\n", + ")\n", "\n", "n_rows = 240\n", "events = (\n", @@ -243,10 +246,10 @@ " \"type\": \"spark\",\n", " \"spark_conf\": {\n", " \"spark.sql.session.timeZone\": \"UTC\",\n", - " \"spark.hadoop.fs.s3a.endpoint\": endpoint.netloc or endpoint.path,\n", + " \"spark.hadoop.fs.s3a.endpoint\": endpoint_host,\n", " \"spark.hadoop.fs.s3a.endpoint.region\": region,\n", " \"spark.hadoop.fs.s3a.path.style.access\": \"true\",\n", - " \"spark.hadoop.fs.s3a.connection.ssl.enabled\": str(endpoint.scheme == \"https\").lower(),\n", + " \"spark.hadoop.fs.s3a.connection.ssl.enabled\": ssl_enabled,\n", " },\n", "}\n", "(repo / \"feature_store.yaml\").write_text(yaml.safe_dump(batch_config, sort_keys=False))\n", diff --git a/e2e/cases/c16_feast_offline_online.sh b/e2e/cases/c16_feast_offline_online.sh index 6ecb4b01..5c147eda 100755 --- a/e2e/cases/c16_feast_offline_online.sh +++ b/e2e/cases/c16_feast_offline_online.sh @@ -19,6 +19,7 @@ SPARK_APP="$FS_NAME-spark" SPARK_SA="$FS_NAME-spark" CM="$FS_NAME-code" S3_KEY="e2e/$FS_NAME/driver_stats" +# The e2e prefix is unique per run; object-store lifecycle policy owns expiry. TMP="$(mktemp -d)" PF="" @@ -123,15 +124,18 @@ model_repo = Path("/mnt/models/repo") repo.mkdir(parents=True, exist_ok=True) model_repo.mkdir(parents=True, exist_ok=True) -spark = SparkSession.builder.appName("feast-offline-online-e2e").getOrCreate() endpoint = urlparse(os.environ["S3_ENDPOINT_URL"]) -hadoop = spark.sparkContext._jsc.hadoopConfiguration() -hadoop.set("fs.s3a.endpoint", endpoint.netloc or endpoint.path) -hadoop.set("fs.s3a.endpoint.region", region) -hadoop.set("fs.s3a.path.style.access", "true") -hadoop.set("fs.s3a.connection.ssl.enabled", str(endpoint.scheme == "https").lower()) -hadoop.set("fs.s3a.access.key", os.environ["AWS_ACCESS_KEY_ID"]) -hadoop.set("fs.s3a.secret.key", os.environ["AWS_SECRET_ACCESS_KEY"]) +endpoint_host = endpoint.netloc or endpoint.path +ssl_enabled = str(endpoint.scheme == "https").lower() +spark = ( + SparkSession.builder + .appName("feast-offline-online-e2e") + .config("spark.hadoop.fs.s3a.endpoint", endpoint_host) + .config("spark.hadoop.fs.s3a.endpoint.region", region) + .config("spark.hadoop.fs.s3a.path.style.access", "true") + .config("spark.hadoop.fs.s3a.connection.ssl.enabled", ssl_enabled) + .getOrCreate() +) events = ( spark.range(240) @@ -155,10 +159,10 @@ batch_config["offline_store"] = { "type": "spark", "spark_conf": { "spark.sql.session.timeZone": "UTC", - "spark.hadoop.fs.s3a.endpoint": endpoint.netloc or endpoint.path, + "spark.hadoop.fs.s3a.endpoint": endpoint_host, "spark.hadoop.fs.s3a.endpoint.region": region, "spark.hadoop.fs.s3a.path.style.access": "true", - "spark.hadoop.fs.s3a.connection.ssl.enabled": str(endpoint.scheme == "https").lower(), + "spark.hadoop.fs.s3a.connection.ssl.enabled": ssl_enabled, }, } (repo / "feature_store.yaml").write_text(yaml.safe_dump(batch_config, sort_keys=False)) @@ -223,10 +227,6 @@ serving_config.pop("offline_store", None) shutil.copy("/opt/feast-batch/server.py", model_repo / "server.py") print(json.dumps({"historical_rows": len(training), "online_rows": len(online)})) -if os.environ.get("CLEANUP_S3") == "true": - jvm = spark.sparkContext._jvm - filesystem = jvm.org.apache.hadoop.fs.FileSystem.get(jvm.java.net.URI.create(dataset_uri), hadoop) - filesystem.delete(jvm.org.apache.hadoop.fs.Path(dataset_uri), True) spark.stop() PY @@ -341,7 +341,6 @@ spec: env: - {name: FEAST_PROJECT, value: $PROJECT} - {name: S3_DATASET_KEY, value: $S3_KEY} - - {name: CLEANUP_S3, value: "true"} envFrom: [{secretRef: {name: $FEAST_S3_CREDENTIALS_SECRET}}] volumeMounts: - {name: batch-code, mountPath: /opt/feast-batch, readOnly: true} From 8b65e2c780a2c19181250921241a6b9f469da086 Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Thu, 20 Aug 2026 11:15:52 +0800 Subject: [PATCH 6/7] docs: move Feast example resources to assets --- .../batch.py | 160 ++++ .../feature-store.yaml | 24 + .../inference-service.yaml | 23 + .../model-inspector.yaml | 18 + .../model-pvc.yaml | 11 + .../namespaces.yaml | 9 + .../server.py | 66 ++ .../serving-runtime.yaml | 35 + .../spark-application.yaml | 71 ++ .../spark-rbac.yaml | 29 + .../feast-offline-to-online-inference.ipynb | 749 ++++-------------- e2e/cases/c16_feast_offline_online.sh | 184 +---- 12 files changed, 595 insertions(+), 784 deletions(-) create mode 100644 docs/en/train/guides/assets/feast-offline-to-online-inference/batch.py create mode 100644 docs/en/train/guides/assets/feast-offline-to-online-inference/feature-store.yaml create mode 100644 docs/en/train/guides/assets/feast-offline-to-online-inference/inference-service.yaml create mode 100644 docs/en/train/guides/assets/feast-offline-to-online-inference/model-inspector.yaml create mode 100644 docs/en/train/guides/assets/feast-offline-to-online-inference/model-pvc.yaml create mode 100644 docs/en/train/guides/assets/feast-offline-to-online-inference/namespaces.yaml create mode 100644 docs/en/train/guides/assets/feast-offline-to-online-inference/server.py create mode 100644 docs/en/train/guides/assets/feast-offline-to-online-inference/serving-runtime.yaml create mode 100644 docs/en/train/guides/assets/feast-offline-to-online-inference/spark-application.yaml create mode 100644 docs/en/train/guides/assets/feast-offline-to-online-inference/spark-rbac.yaml diff --git a/docs/en/train/guides/assets/feast-offline-to-online-inference/batch.py b/docs/en/train/guides/assets/feast-offline-to-online-inference/batch.py new file mode 100644 index 00000000..d3217d5b --- /dev/null +++ b/docs/en/train/guides/assets/feast-offline-to-online-inference/batch.py @@ -0,0 +1,160 @@ +import copy +import json +import os +import shutil +import subprocess +from pathlib import Path +from urllib.parse import urlparse + +import numpy as np +import pandas as pd +import yaml +from feast import FeatureStore +from pyspark.sql import SparkSession, functions as F + +project = os.environ["FEAST_PROJECT"] +bucket = os.environ["S3_BUCKET"] +dataset_key = os.environ["S3_DATASET_KEY"].strip("/") +dataset_uri = f"s3a://{bucket}/{dataset_key}" +region = os.environ["AWS_DEFAULT_REGION"] +repo = Path("/tmp/feast-repo") +model_repo = Path("/mnt/models/repo") +repo.mkdir(parents=True, exist_ok=True) +model_repo.mkdir(parents=True, exist_ok=True) + +endpoint = urlparse(os.environ["S3_ENDPOINT_URL"]) +endpoint_host = endpoint.netloc or endpoint.path +ssl_enabled = str(endpoint.scheme == "https").lower() +spark = ( + SparkSession.builder.appName("feast-offline-batch") + .config("spark.hadoop.fs.s3a.endpoint", endpoint_host) + .config("spark.hadoop.fs.s3a.endpoint.region", region) + .config("spark.hadoop.fs.s3a.path.style.access", "true") + .config("spark.hadoop.fs.s3a.connection.ssl.enabled", ssl_enabled) + .getOrCreate() +) + +n_rows = 240 +events = ( + spark.range(n_rows) + .withColumn("driver_id", (F.col("id") % 12 + 1).cast("long")) + .withColumn( + "event_timestamp", + F.timestamp_seconds(F.lit(1767225600) + F.col("id") * 3600), + ) + .withColumn("created", F.col("event_timestamp") + F.expr("INTERVAL 1 MINUTE")) + .withColumn("conv_rate", (F.lit(0.25) + F.lit(0.55) * F.rand(7)).cast("float")) + .withColumn("acc_rate", (F.lit(0.50) + F.lit(0.45) * F.rand(11)).cast("float")) + .withColumn( + "avg_daily_trips", + F.floor(F.lit(2) + F.lit(18) * F.rand(13)).cast("long"), + ) + .withColumn( + "label", + ( + ( + F.col("conv_rate") * 2 + + F.col("acc_rate") + + F.col("avg_daily_trips") / 20 + ) + > 1.8 + ).cast("long"), + ) + .drop("id") +) +events.repartition(4, "driver_id").write.mode("overwrite").partitionBy( + "driver_id" +).parquet(dataset_uri) + +client_config = yaml.safe_load(Path("/etc/feast/feature_store.yaml").read_text()) +batch_config = copy.deepcopy(client_config) +batch_config["offline_store"] = { + "type": "spark", + "spark_conf": { + "spark.sql.session.timeZone": "UTC", + "spark.hadoop.fs.s3a.endpoint": endpoint_host, + "spark.hadoop.fs.s3a.endpoint.region": region, + "spark.hadoop.fs.s3a.path.style.access": "true", + "spark.hadoop.fs.s3a.connection.ssl.enabled": ssl_enabled, + }, +} +(repo / "feature_store.yaml").write_text( + yaml.safe_dump(batch_config, sort_keys=False) +) +(repo / "features.py").write_text( + f'''from datetime import timedelta +from feast import Entity, FeatureService, FeatureView, Field +from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import SparkSource +from feast.types import Float32, Int64 +from feast.value_type import ValueType + +driver = Entity(name="driver", join_keys=["driver_id"], value_type=ValueType.INT64) +source = SparkSource( + name="driver_stats_source", + path="{dataset_uri}", + file_format="parquet", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) +view = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=365), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=source, +) +driver_activity_v1 = FeatureService(name="driver_activity_v1", features=[view]) +''' +) + +subprocess.run(["feast", "--chdir", str(repo), "apply"], check=True) +store = FeatureStore(repo_path=str(repo)) +entity_df = events.select("driver_id", "event_timestamp", "label").toPandas() +training_df = store.get_historical_features( + entity_df=entity_df, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + ], +).to_df().dropna() +columns = ["conv_rate", "acc_rate", "avg_daily_trips"] +x = training_df[columns].to_numpy(dtype="float64") +y = training_df["label"].to_numpy(dtype="float64") +weights = np.linalg.pinv(np.column_stack([np.ones(len(x)), x])) @ y +np.savez( + "/mnt/models/model.npz", + weights=weights, + feature_columns=np.array(columns), +) + +end_date = events.agg(F.max("event_timestamp")).first()[0] + pd.Timedelta(hours=1) +store.materialize_incremental(end_date) +online = store.get_online_features( + features=store.get_feature_service("driver_activity_v1"), + entity_rows=[{"driver_id": 1}, {"driver_id": 2}], +).to_df() +if len(online) != 2 or online[columns].isna().any().any(): + raise RuntimeError(f"online feature verification failed: {online}") +online.to_json("/mnt/models/online-sample.json", orient="records") +serving_config = copy.deepcopy(client_config) +serving_config.pop("offline_store", None) +(model_repo / "feature_store.yaml").write_text( + yaml.safe_dump(serving_config, sort_keys=False) +) +shutil.copy("/opt/feast-batch/server.py", model_repo / "server.py") +print( + json.dumps( + { + "historical_rows": len(training_df), + "dataset": dataset_uri, + "online_rows": len(online), + } + ) +) +spark.stop() diff --git a/docs/en/train/guides/assets/feast-offline-to-online-inference/feature-store.yaml b/docs/en/train/guides/assets/feast-offline-to-online-inference/feature-store.yaml new file mode 100644 index 00000000..b99d0189 --- /dev/null +++ b/docs/en/train/guides/assets/feast-offline-to-online-inference/feature-store.yaml @@ -0,0 +1,24 @@ +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: feast-notebook + namespace: feast-demo +spec: + feastProject: feast_demo + replicas: 1 + services: + onlineStore: + persistence: + store: + type: redis + secretRef: + name: feast-data-stores + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores + server: {} + ui: {} diff --git a/docs/en/train/guides/assets/feast-offline-to-online-inference/inference-service.yaml b/docs/en/train/guides/assets/feast-offline-to-online-inference/inference-service.yaml new file mode 100644 index 00000000..bdde4796 --- /dev/null +++ b/docs/en/train/guides/assets/feast-offline-to-online-inference/inference-service.yaml @@ -0,0 +1,23 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: feast-online-model + namespace: feast-demo + annotations: + serving.kserve.io/deploymentMode: RawDeployment +spec: + predictor: + model: + modelFormat: + name: feast-numpy + version: "1" + protocolVersion: v2 + runtime: feast-numpy-runtime + storageUri: pvc://feast-notebook-model + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi diff --git a/docs/en/train/guides/assets/feast-offline-to-online-inference/model-inspector.yaml b/docs/en/train/guides/assets/feast-offline-to-online-inference/model-inspector.yaml new file mode 100644 index 00000000..ae3f8c10 --- /dev/null +++ b/docs/en/train/guides/assets/feast-offline-to-online-inference/model-inspector.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: feast-model-inspector + namespace: feast-demo +spec: + restartPolicy: Never + containers: + - name: inspector + image: FEAST_MODEL_IMAGE_PLACEHOLDER + command: [bash, -c, cat /mnt/models/online-sample.json] + volumeMounts: + - name: model + mountPath: /mnt/models + volumes: + - name: model + persistentVolumeClaim: + claimName: feast-notebook-model diff --git a/docs/en/train/guides/assets/feast-offline-to-online-inference/model-pvc.yaml b/docs/en/train/guides/assets/feast-offline-to-online-inference/model-pvc.yaml new file mode 100644 index 00000000..de24509e --- /dev/null +++ b/docs/en/train/guides/assets/feast-offline-to-online-inference/model-pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: feast-notebook-model + namespace: feast-demo +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/docs/en/train/guides/assets/feast-offline-to-online-inference/namespaces.yaml b/docs/en/train/guides/assets/feast-offline-to-online-inference/namespaces.yaml new file mode 100644 index 00000000..5a67efc6 --- /dev/null +++ b/docs/en/train/guides/assets/feast-offline-to-online-inference/namespaces.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: feast-demo +--- +apiVersion: v1 +kind: Namespace +metadata: + name: feast-operator-system diff --git a/docs/en/train/guides/assets/feast-offline-to-online-inference/server.py b/docs/en/train/guides/assets/feast-offline-to-online-inference/server.py new file mode 100644 index 00000000..b83d0426 --- /dev/null +++ b/docs/en/train/guides/assets/feast-offline-to-online-inference/server.py @@ -0,0 +1,66 @@ +import os + +import numpy as np +import uvicorn +from fastapi import Body, FastAPI +from feast import FeatureStore + +MODEL_NAME = os.getenv("MODEL_NAME", "feast-online-model") +weights = np.load("/mnt/models/model.npz")["weights"] +store = FeatureStore(repo_path="/mnt/models/repo") +feature_service = store.get_feature_service("driver_activity_v1") +app = FastAPI() + + +@app.get("/v2/health/live") +@app.get("/v2/health/ready") +def ready(): + return {"ready": True} + + +@app.get("/v2/models/{model_name}") +@app.get("/v2/models/{model_name}/ready") +def model_ready(model_name: str): + return {"name": model_name, "ready": model_name == MODEL_NAME} + + +@app.post("/v2/models/{model_name}/infer") +def infer(model_name: str, payload: dict = Body(...)): + ids = next( + item for item in payload["inputs"] if item["name"] == "driver_id" + )["data"] + rows = [{"driver_id": int(driver_id)} for driver_id in ids] + values = store.get_online_features( + features=feature_service, + entity_rows=rows, + ).to_dict() + + def column(name): + if name in values: + return values[name] + return values[next(key for key in values if key.endswith("__" + name))] + + x = np.column_stack( + [ + np.ones(len(ids)), + column("conv_rate"), + column("acc_rate"), + column("avg_daily_trips"), + ] + ) + prediction = (x @ weights).astype("float32") + return { + "model_name": model_name, + "outputs": [ + { + "name": "prediction", + "shape": [len(ids)], + "datatype": "FP32", + "data": prediction.tolist(), + } + ], + } + + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8080) diff --git a/docs/en/train/guides/assets/feast-offline-to-online-inference/serving-runtime.yaml b/docs/en/train/guides/assets/feast-offline-to-online-inference/serving-runtime.yaml new file mode 100644 index 00000000..5786c35d --- /dev/null +++ b/docs/en/train/guides/assets/feast-offline-to-online-inference/serving-runtime.yaml @@ -0,0 +1,35 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: ServingRuntime +metadata: + name: feast-numpy-runtime + namespace: feast-demo +spec: + containers: + - name: kserve-container + image: FEAST_MODEL_IMAGE_PLACEHOLDER + command: [python, /mnt/models/repo/server.py] + ports: + - containerPort: 8080 + name: http1 + protocol: TCP + env: + - name: MODEL_NAME + value: feast-online-model + volumeMounts: + - name: online-tls + mountPath: /tls/online + readOnly: true + - name: registry-tls + mountPath: /tls/registry + readOnly: true + protocolVersions: [v2] + supportedModelFormats: + - name: feast-numpy + version: "1" + volumes: + - name: online-tls + secret: + secretName: feast-feast-notebook-online-tls + - name: registry-tls + secret: + secretName: feast-feast-notebook-registry-tls diff --git a/docs/en/train/guides/assets/feast-offline-to-online-inference/spark-application.yaml b/docs/en/train/guides/assets/feast-offline-to-online-inference/spark-application.yaml new file mode 100644 index 00000000..4f4e3c0e --- /dev/null +++ b/docs/en/train/guides/assets/feast-offline-to-online-inference/spark-application.yaml @@ -0,0 +1,71 @@ +apiVersion: sparkoperator.k8s.io/v1beta2 +kind: SparkApplication +metadata: + name: feast-offline-batch + namespace: feast-demo +spec: + type: Python + pythonVersion: "3" + mode: cluster + image: FEAST_SPARK_IMAGE_PLACEHOLDER + imagePullPolicy: IfNotPresent + mainApplicationFile: local:///opt/feast-batch/batch.py + sparkVersion: FEAST_SPARK_VERSION_PLACEHOLDER + timeToLiveSeconds: 3600 + restartPolicy: + type: Never + sparkConf: + spark.sql.session.timeZone: UTC + volumes: + - name: batch-code + configMap: + name: feast-offline-batch-code + - name: feast-client + configMap: + name: feast-feast-notebook-client + items: + - key: feature_store.yaml + path: feature_store.yaml + - name: model + persistentVolumeClaim: + claimName: feast-notebook-model + - name: online-tls + secret: + secretName: feast-feast-notebook-online-tls + - name: registry-tls + secret: + secretName: feast-feast-notebook-registry-tls + driver: + cores: 1 + memory: 2g + serviceAccount: feast-spark + env: + - name: FEAST_PROJECT + value: feast_demo + - name: S3_DATASET_KEY + value: datasets/driver_stats + envFrom: + - secretRef: + name: feast-s3-credentials + volumeMounts: + - name: batch-code + mountPath: /opt/feast-batch + readOnly: true + - name: feast-client + mountPath: /etc/feast + readOnly: true + - name: model + mountPath: /mnt/models + - name: online-tls + mountPath: /tls/online + readOnly: true + - name: registry-tls + mountPath: /tls/registry + readOnly: true + executor: + instances: 2 + cores: 1 + memory: 1g + envFrom: + - secretRef: + name: feast-s3-credentials diff --git a/docs/en/train/guides/assets/feast-offline-to-online-inference/spark-rbac.yaml b/docs/en/train/guides/assets/feast-offline-to-online-inference/spark-rbac.yaml new file mode 100644 index 00000000..7559d071 --- /dev/null +++ b/docs/en/train/guides/assets/feast-offline-to-online-inference/spark-rbac.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: feast-spark + namespace: feast-demo +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: feast-spark + namespace: feast-demo +rules: + - apiGroups: [""] + resources: ["pods", "pods/log", "services", "configmaps"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: feast-spark + namespace: feast-demo +subjects: + - kind: ServiceAccount + name: feast-spark + namespace: feast-demo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: feast-spark diff --git a/docs/en/train/guides/feast-offline-to-online-inference.ipynb b/docs/en/train/guides/feast-offline-to-online-inference.ipynb index fa1abc92..378953d9 100644 --- a/docs/en/train/guides/feast-offline-to-online-inference.ipynb +++ b/docs/en/train/guides/feast-offline-to-online-inference.ipynb @@ -8,101 +8,40 @@ "\n", "This notebook runs a production-shaped, CPU-only feature pipeline:\n", "\n", - "1. a Spark Operator `SparkApplication` generates a synthetic batch dataset and stores it as Parquet in S3-compatible object storage;\n", + "1. a Spark Operator `SparkApplication` generates a synthetic batch dataset and stores it as partitioned Parquet in S3-compatible object storage;\n", "2. Spark performs Feast's point-in-time historical join and trains a small NumPy model;\n", - "3. Feast uses PostgreSQL for its durable SQL registry and materializes the latest feature values into Redis;\n", + "3. Feast uses PostgreSQL for its durable SQL registry and materializes the latest feature values into Redis; and\n", "4. KServe deploys a model server that reads Redis-backed online features before predicting.\n", "\n", - "This separation is the recommended starting point for larger workloads: object storage is the durable, versionable offline data layer; operator-managed Spark provides elastic batch compute; PostgreSQL stores Feast metadata rather than the feature history; and Redis serves low-latency online lookups. Feast classifies its Spark offline store as a contributed integration without full test coverage, so qualify it against your scale and upgrade requirements or use a fully supported warehouse while retaining the same S3-and-Spark data pipeline. For production, use highly available PostgreSQL, Redis, and S3-compatible services with TLS, secret rotation, backups, monitoring, retention policies, and scheduled `SparkApplication` materialization runs.\n", + "Object storage is the durable, versionable offline data layer; operator-managed Spark provides elastic batch compute; PostgreSQL stores Feast metadata rather than feature history; and Redis serves low-latency online lookups. Feast classifies its Spark offline store as a contributed integration without full test coverage, so qualify it against your scale and upgrade requirements or use a fully supported warehouse while retaining the same S3-and-Spark data pipeline.\n", + "\n", + "The reusable source code and Kubernetes manifests are in `assets/feast-offline-to-online-inference/`. The command cells below assume that path is relative to the notebook working directory. Set `FEAST_ASSET_DIR` when your Workbench starts in a different directory.\n", "\n", "Prerequisites:\n", "\n", "- the Alauda Spark Operator is installed through OLM and `sparkapplications.sparkoperator.k8s.io` exists;\n", - "- a Spark runtime image containing PySpark, Feast 0.61.x with Spark and Redis support, NumPy, pandas, PyArrow, PyYAML, and a Hadoop S3A connector compatible with the image's Hadoop version;\n", - "- a `feast-data-stores` Secret with `redis` and `sql` keys for the Feast Operator;\n", - "- a `feast-s3-credentials` Secret with `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `S3_ENDPOINT_URL`, and `S3_BUCKET`;\n", - "- a pre-created S3 bucket and a ReadWriteOnce storage class for the sample model PVC; and\n", - "- the Feast and KServe operators.\n", + "- `FEAST_SPARK_IMAGE` is set to a Spark runtime containing PySpark, Feast 0.61.x with Spark and Redis support, NumPy, pandas, PyArrow, PyYAML, and a Hadoop S3A connector compatible with the image's Hadoop version;\n", + "- a `feast-data-stores` Secret in `feast-demo` with `redis` and `sql` keys;\n", + "- a `feast-s3-credentials` Secret in `feast-demo` with `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `S3_ENDPOINT_URL`, and `S3_BUCKET`;\n", + "- a pre-created S3 bucket, a default ReadWriteOnce storage class, and the Feast and KServe operators; and\n", + "- `bash`, `kubectl`, `sed`, and `curl` in the Workbench image.\n", "\n", - "Do not copy an internal registry hostname from this document. Find the registry for your global cluster with:\n", + "Find the global-cluster registry without copying an internal registry hostname from this document:\n", "\n", "```bash\n", "kubectl get configmap global-info -n kube-public -o jsonpath='{.data.registryAddress}'\n", "```\n", "\n", - "In the ACP console, open the container registry associated with that address, choose an approved Spark runtime that satisfies the dependency list above, and set its complete reference in `FEAST_SPARK_IMAGE`. Set `FEAST_MODEL_IMAGE` similarly if the default Feast model-server repository or tag is not mirrored in your global registry.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import os\n", - "import subprocess\n", - "import time\n", - "from pathlib import Path\n", - "\n", - "import requests\n", - "\n", - "NAMESPACE = os.environ.get(\"FEAST_NAMESPACE\", \"feast-demo\")\n", - "FEATURESTORE_NAME = os.environ.get(\"FEAST_FEATURESTORE\", \"feast-notebook\")\n", - "FEAST_PROJECT = os.environ.get(\"FEAST_PROJECT\", \"feast_demo\")\n", - "MODEL_PVC = os.environ.get(\"FEAST_MODEL_PVC\", \"feast-notebook-model\")\n", - "MODEL_RUNTIME = os.environ.get(\"FEAST_MODEL_RUNTIME\", \"feast-numpy-runtime\")\n", - "MODEL_NAME = os.environ.get(\"FEAST_MODEL_NAME\", \"feast-online-model\")\n", - "SPARK_APP = os.environ.get(\"FEAST_SPARK_APPLICATION\", \"feast-offline-batch\")\n", - "SPARK_SERVICE_ACCOUNT = os.environ.get(\"FEAST_SPARK_SERVICE_ACCOUNT\", \"feast-spark\")\n", - "SPARK_VERSION = os.environ.get(\"FEAST_SPARK_VERSION\", \"4.0.1\")\n", - "DATA_STORES_SECRET = os.environ.get(\"FEAST_DATA_STORES_SECRET\", \"feast-data-stores\")\n", - "S3_CREDENTIALS_SECRET = os.environ.get(\"FEAST_S3_CREDENTIALS_SECRET\", \"feast-s3-credentials\")\n", - "S3_DATASET_KEY = os.environ.get(\"FEAST_S3_DATASET_KEY\", \"datasets/driver_stats\")\n", - "def kubectl(*args, input_text=None, check=True):\n", - " result = subprocess.run([\"kubectl\", *args], input=input_text, text=True, capture_output=True)\n", - " if check and result.returncode:\n", - " raise RuntimeError(f\"kubectl {' '.join(args)} failed: {result.stderr}\")\n", - " return result.stdout.strip()\n", - "\n", - "def secret_exists(name):\n", - " return bool(kubectl(\"get\", \"secret\", name, \"-n\", NAMESPACE, \"-o\", \"name\", check=False))\n", - "\n", - "registry_address = kubectl(\n", - " \"get\", \"configmap\", \"global-info\", \"-n\", \"kube-public\",\n", - " \"-o\", \"jsonpath={.data.registryAddress}\",\n", - ")\n", - "if not registry_address:\n", - " raise RuntimeError(\"kube-public/global-info does not contain data.registryAddress\")\n", - "\n", - "SPARK_IMAGE = os.environ.get(\"FEAST_SPARK_IMAGE\")\n", - "if not SPARK_IMAGE:\n", - " raise RuntimeError(\n", - " \"Set FEAST_SPARK_IMAGE to an approved Spark+Feast runtime from the registry \"\n", - " f\"reported by kube-public/global-info ({registry_address}).\"\n", - " )\n", - "MODEL_IMAGE = os.environ.get(\n", - " \"FEAST_MODEL_IMAGE\", f\"{registry_address}/mlops/feast/feature-server:0.61.0\"\n", - ")\n", - "\n", - "if not kubectl(\"get\", \"crd\", \"sparkapplications.sparkoperator.k8s.io\", \"-o\", \"name\", check=False):\n", - " raise RuntimeError(\"Install the OLM Spark Operator before continuing\")\n", - "\n", - "print({\n", - " \"namespace\": NAMESPACE,\n", - " \"featurestore\": FEATURESTORE_NAME,\n", - " \"spark_application\": SPARK_APP,\n", - " \"spark_registry\": registry_address,\n", - "})\n" + "Choose an approved Spark runtime from that registry and set its complete reference in `FEAST_SPARK_IMAGE`. Optionally set `FEAST_SPARK_VERSION` and `FEAST_MODEL_IMAGE`; the commands default to Spark 4.0.1 metadata and the Feast 0.61.0 model-server repository in the discovered registry.\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 1. Prepare PostgreSQL registry and Redis online serving\n", + "## 1. Verify the operator and asset bundle\n", "\n", - "The Feast Operator manages only the control plane and online-serving backends here. PostgreSQL stores the Feast SQL registry; Redis stores materialized online feature values. The Spark driver overrides the generated client configuration with Feast's Spark offline store, so no Feast offline-server pod or separate Spark/Hadoop installation is needed.\n" + "The Spark runtime image contains Spark and its S3A client libraries. No separate Spark or Hadoop service is installed by this example.\n" ] }, { @@ -111,73 +50,26 @@ "metadata": {}, "outputs": [], "source": [ - "for namespace in (NAMESPACE, \"feast-operator-system\"):\n", - " namespace_yaml = kubectl(\"create\", \"namespace\", namespace, \"--dry-run=client\", \"-o\", \"yaml\")\n", - " kubectl(\"apply\", \"-f\", \"-\", input_text=namespace_yaml)\n", - "\n", - "for secret_name in (DATA_STORES_SECRET, S3_CREDENTIALS_SECRET):\n", - " if not secret_exists(secret_name):\n", - " raise RuntimeError(f\"Create Secret {NAMESPACE}/{secret_name} before continuing\")\n", - "\n", - "featurestore_yaml = f\"\"\"\n", - "apiVersion: feast.dev/v1\n", - "kind: FeatureStore\n", - "metadata:\n", - " name: {FEATURESTORE_NAME}\n", - " namespace: {NAMESPACE}\n", - "spec:\n", - " feastProject: {FEAST_PROJECT}\n", - " replicas: 1\n", - " services:\n", - " onlineStore:\n", - " persistence:\n", - " store:\n", - " type: redis\n", - " secretRef:\n", - " name: {DATA_STORES_SECRET}\n", - " registry:\n", - " local:\n", - " persistence:\n", - " store:\n", - " type: sql\n", - " secretRef:\n", - " name: {DATA_STORES_SECRET}\n", - " server: {{}}\n", - " ui: {{}}\n", - "\"\"\"\n", - "kubectl(\"apply\", \"-f\", \"-\", input_text=featurestore_yaml)\n", - "\n", - "deadline = time.time() + 600\n", - "while time.time() < deadline:\n", - " phase = kubectl(\n", - " \"get\", \"featurestore\", FEATURESTORE_NAME, \"-n\", NAMESPACE,\n", - " \"-o\", \"jsonpath={.status.phase}\", check=False,\n", - " )\n", - " print(phase or \"Pending\")\n", - " if phase == \"Ready\":\n", - " break\n", - " if phase == \"Failed\":\n", - " raise RuntimeError(kubectl(\"describe\", \"featurestore\", FEATURESTORE_NAME, \"-n\", NAMESPACE, check=False))\n", - " time.sleep(10)\n", - "else:\n", - " raise TimeoutError(\"FeatureStore did not become Ready\")\n", - "\n", - "client_config_map = kubectl(\n", - " \"get\", \"featurestore\", FEATURESTORE_NAME, \"-n\", NAMESPACE,\n", - " \"-o\", \"jsonpath={.status.clientConfigMap}\",\n", - ")\n", - "print(\"FeatureStore is Ready; client ConfigMap:\", client_config_map)\n" + "%%bash\n", + "set -euo pipefail\n", + "ASSET_DIR=\"${FEAST_ASSET_DIR:-assets/feast-offline-to-online-inference}\"\n", + "test -f \"$ASSET_DIR/batch.py\"\n", + "test -f \"$ASSET_DIR/spark-application.yaml\"\n", + ": \"${FEAST_SPARK_IMAGE:?Set FEAST_SPARK_IMAGE to an approved global-registry image}\"\n", + "kubectl get crd sparkapplications.sparkoperator.k8s.io\n", + "kubectl get crd featurestores.feast.dev\n", + "kubectl get crd inferenceservices.serving.kserve.io\n", + "kubectl get configmap global-info -n kube-public -o jsonpath='{.data.registryAddress}'\n", + "echo\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 2. Prepare the synthetic S3 batch application\n", + "## 2. Prepare PostgreSQL registry and Redis online serving\n", "\n", - "This section creates the code that the Spark Operator will mount into the driver. The application generates deterministic synthetic driver events with Spark, writes partitioned Parquet to the configured S3-compatible service, registers a Feast `SparkSource`, performs the point-in-time historical join, trains the sample model, and materializes the same features into Redis.\n", - "\n", - "The S3 endpoint and credentials come from a Kubernetes Secret. The code configures S3A through the public `SparkSession.builder.config()` API before the Spark context is created, while S3A reads the AWS credentials from the driver and executor environments. It enables path-style access so it works with SeaweedFS and similar S3-compatible services; use your service's TLS endpoint in production.\n" + "The Feast Operator manages the control plane and online-serving backends. PostgreSQL stores the SQL registry and Redis stores materialized online feature values. Spark reads the offline Parquet history directly from S3.\n" ] }, { @@ -186,186 +78,38 @@ "metadata": {}, "outputs": [], "source": [ - "batch_source = r'''import copy\n", - "import json\n", - "import os\n", - "import shutil\n", - "import subprocess\n", - "from pathlib import Path\n", - "from urllib.parse import urlparse\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import yaml\n", - "from feast import FeatureStore\n", - "from pyspark.sql import SparkSession, functions as F\n", - "\n", - "project = os.environ[\"FEAST_PROJECT\"]\n", - "bucket = os.environ[\"S3_BUCKET\"]\n", - "dataset_key = os.environ[\"S3_DATASET_KEY\"].strip(\"/\")\n", - "dataset_uri = f\"s3a://{bucket}/{dataset_key}\"\n", - "region = os.environ[\"AWS_DEFAULT_REGION\"]\n", - "repo = Path(\"/tmp/feast-repo\")\n", - "model_repo = Path(\"/mnt/models/repo\")\n", - "repo.mkdir(parents=True, exist_ok=True)\n", - "model_repo.mkdir(parents=True, exist_ok=True)\n", - "\n", - "endpoint = urlparse(os.environ[\"S3_ENDPOINT_URL\"])\n", - "endpoint_host = endpoint.netloc or endpoint.path\n", - "ssl_enabled = str(endpoint.scheme == \"https\").lower()\n", - "spark = (\n", - " SparkSession.builder\n", - " .appName(\"feast-offline-batch\")\n", - " .config(\"spark.hadoop.fs.s3a.endpoint\", endpoint_host)\n", - " .config(\"spark.hadoop.fs.s3a.endpoint.region\", region)\n", - " .config(\"spark.hadoop.fs.s3a.path.style.access\", \"true\")\n", - " .config(\"spark.hadoop.fs.s3a.connection.ssl.enabled\", ssl_enabled)\n", - " .getOrCreate()\n", - ")\n", - "\n", - "n_rows = 240\n", - "events = (\n", - " spark.range(n_rows)\n", - " .withColumn(\"driver_id\", (F.col(\"id\") % 12 + 1).cast(\"long\"))\n", - " .withColumn(\"event_timestamp\", F.timestamp_seconds(F.lit(1767225600) + F.col(\"id\") * 3600))\n", - " .withColumn(\"created\", F.col(\"event_timestamp\") + F.expr(\"INTERVAL 1 MINUTE\"))\n", - " .withColumn(\"conv_rate\", (F.lit(0.25) + F.lit(0.55) * F.rand(7)).cast(\"float\"))\n", - " .withColumn(\"acc_rate\", (F.lit(0.50) + F.lit(0.45) * F.rand(11)).cast(\"float\"))\n", - " .withColumn(\"avg_daily_trips\", F.floor(F.lit(2) + F.lit(18) * F.rand(13)).cast(\"long\"))\n", - " .withColumn(\n", - " \"label\",\n", - " ((F.col(\"conv_rate\") * 2 + F.col(\"acc_rate\") + F.col(\"avg_daily_trips\") / 20) > 1.8).cast(\"long\"),\n", - " )\n", - " .drop(\"id\")\n", - ")\n", - "events.repartition(4, \"driver_id\").write.mode(\"overwrite\").partitionBy(\"driver_id\").parquet(dataset_uri)\n", - "\n", - "client_config = yaml.safe_load(Path(\"/etc/feast/feature_store.yaml\").read_text())\n", - "batch_config = copy.deepcopy(client_config)\n", - "batch_config[\"offline_store\"] = {\n", - " \"type\": \"spark\",\n", - " \"spark_conf\": {\n", - " \"spark.sql.session.timeZone\": \"UTC\",\n", - " \"spark.hadoop.fs.s3a.endpoint\": endpoint_host,\n", - " \"spark.hadoop.fs.s3a.endpoint.region\": region,\n", - " \"spark.hadoop.fs.s3a.path.style.access\": \"true\",\n", - " \"spark.hadoop.fs.s3a.connection.ssl.enabled\": ssl_enabled,\n", - " },\n", - "}\n", - "(repo / \"feature_store.yaml\").write_text(yaml.safe_dump(batch_config, sort_keys=False))\n", - "(repo / \"features.py\").write_text(f\"\"\"from datetime import timedelta\n", - "from feast import Entity, FeatureService, FeatureView, Field\n", - "from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import SparkSource\n", - "from feast.types import Float32, Int64\n", - "from feast.value_type import ValueType\n", - "\n", - "driver = Entity(name=\"driver\", join_keys=[\"driver_id\"], value_type=ValueType.INT64)\n", - "source = SparkSource(\n", - " name=\"driver_stats_source\",\n", - " path=\"{dataset_uri}\",\n", - " file_format=\"parquet\",\n", - " timestamp_field=\"event_timestamp\",\n", - " created_timestamp_column=\"created\",\n", - ")\n", - "view = FeatureView(\n", - " name=\"driver_hourly_stats\",\n", - " entities=[driver],\n", - " ttl=timedelta(days=365),\n", - " schema=[\n", - " Field(name=\"conv_rate\", dtype=Float32),\n", - " Field(name=\"acc_rate\", dtype=Float32),\n", - " Field(name=\"avg_daily_trips\", dtype=Int64),\n", - " ],\n", - " online=True,\n", - " source=source,\n", - ")\n", - "driver_activity_v1 = FeatureService(name=\"driver_activity_v1\", features=[view])\n", - "\"\"\")\n", - "\n", - "subprocess.run([\"feast\", \"--chdir\", str(repo), \"apply\"], check=True)\n", - "store = FeatureStore(repo_path=str(repo))\n", - "entity_df = events.select(\"driver_id\", \"event_timestamp\", \"label\").toPandas()\n", - "training_df = store.get_historical_features(\n", - " entity_df=entity_df,\n", - " features=[\n", - " \"driver_hourly_stats:conv_rate\",\n", - " \"driver_hourly_stats:acc_rate\",\n", - " \"driver_hourly_stats:avg_daily_trips\",\n", - " ],\n", - ").to_df().dropna()\n", - "columns = [\"conv_rate\", \"acc_rate\", \"avg_daily_trips\"]\n", - "x = training_df[columns].to_numpy(dtype=\"float64\")\n", - "y = training_df[\"label\"].to_numpy(dtype=\"float64\")\n", - "weights = np.linalg.pinv(np.column_stack([np.ones(len(x)), x])) @ y\n", - "np.savez(\"/mnt/models/model.npz\", weights=weights, feature_columns=np.array(columns))\n", - "\n", - "store.materialize_incremental(events.agg(F.max(\"event_timestamp\")).first()[0] + pd.Timedelta(hours=1))\n", - "online = store.get_online_features(\n", - " features=store.get_feature_service(\"driver_activity_v1\"),\n", - " entity_rows=[{\"driver_id\": 1}, {\"driver_id\": 2}],\n", - ").to_df()\n", - "online.to_json(\"/mnt/models/online-sample.json\", orient=\"records\")\n", - "serving_config = copy.deepcopy(client_config)\n", - "serving_config.pop(\"offline_store\", None)\n", - "(model_repo / \"feature_store.yaml\").write_text(yaml.safe_dump(serving_config, sort_keys=False))\n", - "shutil.copy(\"/opt/feast-batch/server.py\", model_repo / \"server.py\")\n", - "print(json.dumps({\"rows\": len(training_df), \"dataset\": dataset_uri, \"online_rows\": len(online)}))\n", - "spark.stop()\n", - "'''\n", - "\n", - "server_source = r'''import os\n", - "import numpy as np\n", - "from fastapi import Body, FastAPI\n", - "from feast import FeatureStore\n", - "import uvicorn\n", - "\n", - "MODEL_NAME = os.getenv(\"MODEL_NAME\", \"feast-online-model\")\n", - "weights = np.load(\"/mnt/models/model.npz\")[\"weights\"]\n", - "store = FeatureStore(repo_path=\"/mnt/models/repo\")\n", - "feature_service = store.get_feature_service(\"driver_activity_v1\")\n", - "app = FastAPI()\n", - "\n", - "@app.get(\"/v2/health/live\")\n", - "@app.get(\"/v2/health/ready\")\n", - "def ready():\n", - " return {\"ready\": True}\n", - "\n", - "@app.get(\"/v2/models/{model_name}\")\n", - "@app.get(\"/v2/models/{model_name}/ready\")\n", - "def model_ready(model_name: str):\n", - " return {\"name\": model_name, \"ready\": model_name == MODEL_NAME}\n", - "\n", - "@app.post(\"/v2/models/{model_name}/infer\")\n", - "def infer(model_name: str, payload: dict = Body(...)):\n", - " ids = next(item for item in payload[\"inputs\"] if item[\"name\"] == \"driver_id\")[\"data\"]\n", - " rows = [{\"driver_id\": int(driver_id)} for driver_id in ids]\n", - " values = store.get_online_features(features=feature_service, entity_rows=rows).to_dict()\n", - " def column(name):\n", - " if name in values:\n", - " return values[name]\n", - " return values[next(key for key in values if key.endswith(\"__\" + name))]\n", - " x = np.column_stack([np.ones(len(ids)), column(\"conv_rate\"), column(\"acc_rate\"), column(\"avg_daily_trips\")])\n", - " prediction = (x @ weights).astype(\"float32\")\n", - " return {\"model_name\": model_name, \"outputs\": [{\"name\": \"prediction\", \"shape\": [len(ids)], \"datatype\": \"FP32\", \"data\": prediction.tolist()}]}\n", - "\n", - "if __name__ == \"__main__\":\n", - " uvicorn.run(app, host=\"0.0.0.0\", port=8080)\n", - "'''\n", - "\n", - "batch_dir = Path(\"feast-spark-batch\")\n", - "batch_dir.mkdir(exist_ok=True)\n", - "(batch_dir / \"batch.py\").write_text(batch_source)\n", - "(batch_dir / \"server.py\").write_text(server_source)\n", - "print(\"Prepared\", batch_dir)\n" + "%%bash\n", + "set -euo pipefail\n", + "ASSET_DIR=\"${FEAST_ASSET_DIR:-assets/feast-offline-to-online-inference}\"\n", + "kubectl apply -f \"$ASSET_DIR/namespaces.yaml\"\n", + "kubectl -n feast-demo get secret feast-data-stores\n", + "kubectl -n feast-demo get secret feast-s3-credentials\n", + "kubectl apply -f \"$ASSET_DIR/feature-store.yaml\"\n", + "\n", + "for _ in $(seq 1 60); do\n", + " phase=\"$(kubectl -n feast-demo get featurestore feast-notebook \\\n", + " -o jsonpath='{.status.phase}' 2>/dev/null || true)\"\n", + " echo \"${phase:-Pending}\"\n", + " [ \"$phase\" = Ready ] && break\n", + " if [ \"$phase\" = Failed ]; then\n", + " kubectl -n feast-demo describe featurestore feast-notebook\n", + " exit 1\n", + " fi\n", + " sleep 10\n", + "done\n", + "[ \"${phase:-}\" = Ready ] || { echo \"FeatureStore did not become Ready\" >&2; exit 1; }\n", + "kubectl -n feast-demo get featurestore feast-notebook\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 3. Submit the batch work as a SparkApplication\n", + "## 3. Submit the synthetic S3 batch as a SparkApplication\n", + "\n", + "The code ConfigMap packages `batch.py` and `server.py` from the asset directory. The Spark driver configures S3A through the public `SparkSession.builder.config()` API before creating the Spark context. S3A reads credentials from the Secret-backed driver and executor environments, so the keys are not written into the `SparkApplication` or model PVC.\n", "\n", - "The Spark Operator creates and monitors the driver and executor pods from this CR. A namespace-scoped service account gives the driver only the permissions it needs to manage its executors. The model PVC is mounted only in the driver; the dataset remains in S3.\n" + "The application generates deterministic synthetic driver events, writes partitioned Parquet to S3, registers a Feast `SparkSource`, performs the historical join, trains the sample model, and materializes the same features into Redis.\n" ] }, { @@ -374,160 +118,42 @@ "metadata": {}, "outputs": [], "source": [ - "rbac_yaml = f\"\"\"\n", - "apiVersion: v1\n", - "kind: ServiceAccount\n", - "metadata:\n", - " name: {SPARK_SERVICE_ACCOUNT}\n", - " namespace: {NAMESPACE}\n", - "---\n", - "apiVersion: rbac.authorization.k8s.io/v1\n", - "kind: Role\n", - "metadata:\n", - " name: {SPARK_SERVICE_ACCOUNT}\n", - " namespace: {NAMESPACE}\n", - "rules:\n", - "- apiGroups: [\"\"]\n", - " resources: [\"pods\", \"pods/log\", \"services\", \"configmaps\"]\n", - " verbs: [\"get\", \"list\", \"watch\", \"create\", \"delete\", \"patch\"]\n", - "---\n", - "apiVersion: rbac.authorization.k8s.io/v1\n", - "kind: RoleBinding\n", - "metadata:\n", - " name: {SPARK_SERVICE_ACCOUNT}\n", - " namespace: {NAMESPACE}\n", - "subjects:\n", - "- kind: ServiceAccount\n", - " name: {SPARK_SERVICE_ACCOUNT}\n", - " namespace: {NAMESPACE}\n", - "roleRef:\n", - " apiGroup: rbac.authorization.k8s.io\n", - " kind: Role\n", - " name: {SPARK_SERVICE_ACCOUNT}\n", - "\"\"\"\n", - "kubectl(\"apply\", \"-f\", \"-\", input_text=rbac_yaml)\n", - "\n", - "pvc_yaml = f\"\"\"\n", - "apiVersion: v1\n", - "kind: PersistentVolumeClaim\n", - "metadata:\n", - " name: {MODEL_PVC}\n", - " namespace: {NAMESPACE}\n", - "spec:\n", - " accessModes: [ReadWriteOnce]\n", - " resources:\n", - " requests:\n", - " storage: 1Gi\n", - "\"\"\"\n", - "kubectl(\"apply\", \"-f\", \"-\", input_text=pvc_yaml)\n", - "\n", - "batch_config_map = f\"{SPARK_APP}-code\"\n", - "config_map_yaml = kubectl(\n", - " \"create\", \"configmap\", batch_config_map, \"-n\", NAMESPACE,\n", - " f\"--from-file=batch.py={Path('feast-spark-batch/batch.py')}\",\n", - " f\"--from-file=server.py={Path('feast-spark-batch/server.py')}\",\n", - " \"--dry-run=client\", \"-o\", \"yaml\",\n", - ")\n", - "kubectl(\"apply\", \"-f\", \"-\", input_text=config_map_yaml)\n", - "\n", - "spark_application_yaml = f\"\"\"\n", - "apiVersion: sparkoperator.k8s.io/v1beta2\n", - "kind: SparkApplication\n", - "metadata:\n", - " name: {SPARK_APP}\n", - " namespace: {NAMESPACE}\n", - "spec:\n", - " type: Python\n", - " mode: cluster\n", - " image: {SPARK_IMAGE}\n", - " imagePullPolicy: IfNotPresent\n", - " mainApplicationFile: local:///opt/feast-batch/batch.py\n", - " sparkVersion: {SPARK_VERSION}\n", - " timeToLiveSeconds: 3600\n", - " restartPolicy:\n", - " type: Never\n", - " sparkConf:\n", - " spark.sql.session.timeZone: UTC\n", - " volumes:\n", - " - name: batch-code\n", - " configMap:\n", - " name: {batch_config_map}\n", - " - name: feast-client\n", - " configMap:\n", - " name: {client_config_map}\n", - " items:\n", - " - key: feature_store.yaml\n", - " path: feature_store.yaml\n", - " - name: model\n", - " persistentVolumeClaim:\n", - " claimName: {MODEL_PVC}\n", - " - name: online-tls\n", - " secret:\n", - " secretName: feast-{FEATURESTORE_NAME}-online-tls\n", - " - name: registry-tls\n", - " secret:\n", - " secretName: feast-{FEATURESTORE_NAME}-registry-tls\n", - " driver:\n", - " cores: 1\n", - " memory: 2g\n", - " serviceAccount: {SPARK_SERVICE_ACCOUNT}\n", - " env:\n", - " - name: FEAST_PROJECT\n", - " value: {FEAST_PROJECT}\n", - " - name: S3_DATASET_KEY\n", - " value: {S3_DATASET_KEY}\n", - " envFrom:\n", - " - secretRef:\n", - " name: {S3_CREDENTIALS_SECRET}\n", - " volumeMounts:\n", - " - name: batch-code\n", - " mountPath: /opt/feast-batch\n", - " readOnly: true\n", - " - name: feast-client\n", - " mountPath: /etc/feast\n", - " readOnly: true\n", - " - name: model\n", - " mountPath: /mnt/models\n", - " - name: online-tls\n", - " mountPath: /tls/online\n", - " readOnly: true\n", - " - name: registry-tls\n", - " mountPath: /tls/registry\n", - " readOnly: true\n", - " executor:\n", - " instances: 2\n", - " cores: 1\n", - " memory: 1g\n", - " envFrom:\n", - " - secretRef:\n", - " name: {S3_CREDENTIALS_SECRET}\n", - "\"\"\"\n", - "kubectl(\"delete\", \"sparkapplication\", SPARK_APP, \"-n\", NAMESPACE, \"--ignore-not-found\", \"--wait=true\")\n", - "kubectl(\"apply\", \"-f\", \"-\", input_text=spark_application_yaml)\n", - "\n", - "deadline = time.time() + 1200\n", - "terminal = {\"COMPLETED\", \"FAILED\", \"FAILED_SUBMISSION\", \"INVALIDATING\", \"UNKNOWN\"}\n", - "while time.time() < deadline:\n", - " state = kubectl(\n", - " \"get\", \"sparkapplication\", SPARK_APP, \"-n\", NAMESPACE,\n", - " \"-o\", \"jsonpath={.status.applicationState.state}\", check=False,\n", - " )\n", - " print(state or \"SUBMITTED\")\n", - " if state in terminal:\n", - " break\n", - " time.sleep(10)\n", - "else:\n", - " raise TimeoutError(\"SparkApplication did not reach a terminal state\")\n", - "\n", - "if state != \"COMPLETED\":\n", - " driver_pod = kubectl(\n", - " \"get\", \"sparkapplication\", SPARK_APP, \"-n\", NAMESPACE,\n", - " \"-o\", \"jsonpath={.status.driverInfo.podName}\", check=False,\n", - " )\n", - " logs = kubectl(\"logs\", driver_pod, \"-n\", NAMESPACE, \"--tail=300\", check=False) if driver_pod else \"\"\n", - " raise RuntimeError(f\"SparkApplication ended in {state}\\n{logs}\")\n", - "\n", - "print(\"Spark batch completed\")\n" + "%%bash\n", + "set -euo pipefail\n", + "ASSET_DIR=\"${FEAST_ASSET_DIR:-assets/feast-offline-to-online-inference}\"\n", + ": \"${FEAST_SPARK_IMAGE:?Set FEAST_SPARK_IMAGE to an approved global-registry image}\"\n", + "SPARK_VERSION=\"${FEAST_SPARK_VERSION:-4.0.1}\"\n", + "\n", + "kubectl -n feast-demo create configmap feast-offline-batch-code \\\n", + " --from-file=batch.py=\"$ASSET_DIR/batch.py\" \\\n", + " --from-file=server.py=\"$ASSET_DIR/server.py\" \\\n", + " --dry-run=client -o yaml | kubectl apply -f -\n", + "kubectl apply -f \"$ASSET_DIR/spark-rbac.yaml\"\n", + "kubectl apply -f \"$ASSET_DIR/model-pvc.yaml\"\n", + "kubectl -n feast-demo delete sparkapplication feast-offline-batch \\\n", + " --ignore-not-found --wait=true\n", + "sed \\\n", + " -e \"s|FEAST_SPARK_IMAGE_PLACEHOLDER|$FEAST_SPARK_IMAGE|g\" \\\n", + " -e \"s|FEAST_SPARK_VERSION_PLACEHOLDER|$SPARK_VERSION|g\" \\\n", + " \"$ASSET_DIR/spark-application.yaml\" | kubectl apply -f -\n", + "\n", + "state=\"\"\n", + "for _ in $(seq 1 120); do\n", + " state=\"$(kubectl -n feast-demo get sparkapplication feast-offline-batch \\\n", + " -o jsonpath='{.status.applicationState.state}' 2>/dev/null || true)\"\n", + " echo \"${state:-SUBMITTED}\"\n", + " case \"$state\" in\n", + " COMPLETED|FAILED|FAILED_SUBMISSION|INVALIDATING|UNKNOWN) break ;;\n", + " esac\n", + " sleep 10\n", + "done\n", + "driver=\"$(kubectl -n feast-demo get sparkapplication feast-offline-batch \\\n", + " -o jsonpath='{.status.driverInfo.podName}' 2>/dev/null || true)\"\n", + "if [ \"$state\" != COMPLETED ]; then\n", + " [ -n \"$driver\" ] && kubectl -n feast-demo logs \"$driver\" --tail=300 || true\n", + " exit 1\n", + "fi\n", + "[ -n \"$driver\" ] && kubectl -n feast-demo logs \"$driver\" --tail=100\n" ] }, { @@ -536,7 +162,7 @@ "source": [ "## 4. Inspect the offline-to-online result\n", "\n", - "The driver writes a small verification sample to the model PVC after materialization. Inspecting it through a short-lived pod confirms that the Spark batch completed the S3 historical path and that Feast could read the materialized Redis values.\n" + "The completed Spark driver writes a small online-feature verification sample and the model artifact to the PVC. This short-lived inspector reads the sample without embedding it in the notebook.\n" ] }, { @@ -545,40 +171,30 @@ "metadata": {}, "outputs": [], "source": [ - "inspector_yaml = f\"\"\"\n", - "apiVersion: v1\n", - "kind: Pod\n", - "metadata:\n", - " name: feast-model-inspector\n", - " namespace: {NAMESPACE}\n", - "spec:\n", - " restartPolicy: Never\n", - " containers:\n", - " - name: inspector\n", - " image: {MODEL_IMAGE}\n", - " command: [bash, -c, cat /mnt/models/online-sample.json]\n", - " volumeMounts:\n", - " - name: model\n", - " mountPath: /mnt/models\n", - " volumes:\n", - " - name: model\n", - " persistentVolumeClaim:\n", - " claimName: {MODEL_PVC}\n", - "\"\"\"\n", - "kubectl(\"delete\", \"pod\", \"feast-model-inspector\", \"-n\", NAMESPACE, \"--ignore-not-found\", \"--wait=true\")\n", - "kubectl(\"apply\", \"-f\", \"-\", input_text=inspector_yaml)\n", - "kubectl(\"wait\", \"--for=jsonpath={.status.phase}=Succeeded\", \"pod/feast-model-inspector\", \"-n\", NAMESPACE, \"--timeout=180s\")\n", - "print(kubectl(\"logs\", \"feast-model-inspector\", \"-n\", NAMESPACE))\n", - "kubectl(\"delete\", \"pod\", \"feast-model-inspector\", \"-n\", NAMESPACE, \"--wait=true\")\n" + "%%bash\n", + "set -euo pipefail\n", + "ASSET_DIR=\"${FEAST_ASSET_DIR:-assets/feast-offline-to-online-inference}\"\n", + "MODEL_IMAGE=\"${FEAST_MODEL_IMAGE:-}\"\n", + "if [ -z \"$MODEL_IMAGE\" ]; then\n", + " registry=\"$(kubectl get configmap global-info -n kube-public -o jsonpath='{.data.registryAddress}')\"\n", + " MODEL_IMAGE=\"$registry/mlops/feast/feature-server:0.61.0\"\n", + "fi\n", + "kubectl -n feast-demo delete pod feast-model-inspector --ignore-not-found --wait=true\n", + "sed \"s|FEAST_MODEL_IMAGE_PLACEHOLDER|$MODEL_IMAGE|g\" \\\n", + " \"$ASSET_DIR/model-inspector.yaml\" | kubectl apply -f -\n", + "kubectl -n feast-demo wait \\\n", + " \"--for=jsonpath={.status.phase}=Succeeded\" pod/feast-model-inspector --timeout=180s\n", + "kubectl -n feast-demo logs feast-model-inspector\n", + "kubectl -n feast-demo delete pod feast-model-inspector --wait=true\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 5. Deploy the online model with KServe\n", + "## 5. Deploy the Redis-backed model with KServe\n", "\n", - "The model server loads the weights and Feast definitions from the PVC, queries Redis through the operator-managed Feast online service, and exposes KServe's v2 inference protocol. The online and registry TLS Secrets are mounted at the paths in the generated Feast client configuration.\n" + "The model server loads the weights and serving-only Feast configuration from the PVC, queries Redis through the operator-managed Feast online service, and exposes KServe's v2 inference protocol.\n" ] }, { @@ -587,71 +203,18 @@ "metadata": {}, "outputs": [], "source": [ - "runtime_yaml = f\"\"\"\n", - "apiVersion: serving.kserve.io/v1alpha1\n", - "kind: ServingRuntime\n", - "metadata:\n", - " name: {MODEL_RUNTIME}\n", - " namespace: {NAMESPACE}\n", - "spec:\n", - " containers:\n", - " - name: kserve-container\n", - " image: {MODEL_IMAGE}\n", - " command: [python, /mnt/models/repo/server.py]\n", - " ports:\n", - " - containerPort: 8080\n", - " name: http1\n", - " protocol: TCP\n", - " env:\n", - " - name: MODEL_NAME\n", - " value: {MODEL_NAME}\n", - " volumeMounts:\n", - " - name: online-tls\n", - " mountPath: /tls/online\n", - " readOnly: true\n", - " - name: registry-tls\n", - " mountPath: /tls/registry\n", - " readOnly: true\n", - " protocolVersions: [v2]\n", - " supportedModelFormats:\n", - " - name: feast-numpy\n", - " version: \"1\"\n", - " volumes:\n", - " - name: online-tls\n", - " secret:\n", - " secretName: feast-{FEATURESTORE_NAME}-online-tls\n", - " - name: registry-tls\n", - " secret:\n", - " secretName: feast-{FEATURESTORE_NAME}-registry-tls\n", - "\"\"\"\n", - "isvc_yaml = f\"\"\"\n", - "apiVersion: serving.kserve.io/v1beta1\n", - "kind: InferenceService\n", - "metadata:\n", - " name: {MODEL_NAME}\n", - " namespace: {NAMESPACE}\n", - " annotations:\n", - " serving.kserve.io/deploymentMode: RawDeployment\n", - "spec:\n", - " predictor:\n", - " model:\n", - " modelFormat:\n", - " name: feast-numpy\n", - " version: \"1\"\n", - " protocolVersion: v2\n", - " runtime: {MODEL_RUNTIME}\n", - " storageUri: pvc://{MODEL_PVC}\n", - " resources:\n", - " requests:\n", - " cpu: \"100m\"\n", - " memory: 256Mi\n", - " limits:\n", - " cpu: \"1\"\n", - " memory: 1Gi\n", - "\"\"\"\n", - "kubectl(\"apply\", \"-f\", \"-\", input_text=runtime_yaml)\n", - "kubectl(\"apply\", \"-f\", \"-\", input_text=isvc_yaml)\n", - "print(kubectl(\"get\", \"inferenceservice\", MODEL_NAME, \"-n\", NAMESPACE))\n" + "%%bash\n", + "set -euo pipefail\n", + "ASSET_DIR=\"${FEAST_ASSET_DIR:-assets/feast-offline-to-online-inference}\"\n", + "MODEL_IMAGE=\"${FEAST_MODEL_IMAGE:-}\"\n", + "if [ -z \"$MODEL_IMAGE\" ]; then\n", + " registry=\"$(kubectl get configmap global-info -n kube-public -o jsonpath='{.data.registryAddress}')\"\n", + " MODEL_IMAGE=\"$registry/mlops/feast/feature-server:0.61.0\"\n", + "fi\n", + "sed \"s|FEAST_MODEL_IMAGE_PLACEHOLDER|$MODEL_IMAGE|g\" \\\n", + " \"$ASSET_DIR/serving-runtime.yaml\" | kubectl apply -f -\n", + "kubectl apply -f \"$ASSET_DIR/inference-service.yaml\"\n", + "kubectl -n feast-demo get inferenceservice feast-online-model\n" ] }, { @@ -660,7 +223,7 @@ "source": [ "## 6. Send an online-feature prediction\n", "\n", - "After the predictor has an available replica, send driver IDs to the KServe v2 endpoint. The server looks up their materialized features in Redis and combines those values with the model trained by the `SparkApplication`.\n" + "After the predictor has an available replica, send driver IDs to the KServe v2 endpoint. The server retrieves their materialized features from Redis and combines those values with the model trained by the `SparkApplication`.\n" ] }, { @@ -669,55 +232,31 @@ "metadata": {}, "outputs": [], "source": [ - "deadline = time.time() + 900\n", - "predictor_deployment = f\"{MODEL_NAME}-predictor\"\n", - "while time.time() < deadline:\n", - " deployment_text = kubectl(\n", - " \"get\", \"deployment\", predictor_deployment, \"-n\", NAMESPACE, \"-o\", \"json\", check=False\n", - " )\n", - " deployment = json.loads(deployment_text) if deployment_text else {}\n", - " available = deployment.get(\"status\", {}).get(\"availableReplicas\", 0) or 0\n", - " print({\"availableReplicas\": available})\n", - " if available >= 1:\n", - " break\n", - " time.sleep(10)\n", - "else:\n", - " raise TimeoutError(\"KServe predictor deployment did not become available\")\n", - "\n", - "port_forward = subprocess.Popen(\n", - " [\"kubectl\", \"port-forward\", f\"service/{predictor_deployment}\", \"18080:80\", \"-n\", NAMESPACE],\n", - " stdout=subprocess.DEVNULL,\n", - " stderr=subprocess.DEVNULL,\n", - ")\n", - "base_url = \"http://127.0.0.1:18080\"\n", - "try:\n", - " deadline = time.time() + 60\n", - " while time.time() < deadline:\n", - " if port_forward.poll() is not None:\n", - " raise RuntimeError(\"kubectl port-forward exited unexpectedly\")\n", - " try:\n", - " if requests.get(f\"{base_url}/v2/health/ready\", timeout=2).ok:\n", - " break\n", - " except requests.RequestException:\n", - " pass\n", - " time.sleep(2)\n", - " else:\n", - " raise TimeoutError(\"KServe predictor endpoint did not become ready\")\n", - "\n", - " response = requests.post(\n", - " f\"{base_url}/v2/models/{MODEL_NAME}/infer\",\n", - " json={\"inputs\": [{\"name\": \"driver_id\", \"shape\": [2], \"datatype\": \"INT64\", \"data\": [1, 2]}]},\n", - " timeout=30,\n", - " )\n", - " response.raise_for_status()\n", - " print(json.dumps(response.json(), indent=2))\n", - "finally:\n", - " port_forward.terminate()\n", - " try:\n", - " port_forward.wait(timeout=5)\n", - " except subprocess.TimeoutExpired:\n", - " port_forward.kill()\n", - " port_forward.wait()\n" + "%%bash\n", + "set -euo pipefail\n", + "available=\"\"\n", + "for _ in $(seq 1 90); do\n", + " available=\"$(kubectl -n feast-demo get deployment feast-online-model-predictor \\\n", + " -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)\"\n", + " echo \"availableReplicas=${available:-0}\"\n", + " [ \"${available:-0}\" -ge 1 ] 2>/dev/null && break\n", + " sleep 10\n", + "done\n", + "[ \"${available:-0}\" -ge 1 ] 2>/dev/null || exit 1\n", + "\n", + "kubectl -n feast-demo port-forward service/feast-online-model-predictor 18080:80 \\\n", + " >/tmp/feast-model-port-forward.log 2>&1 &\n", + "port_forward_pid=$!\n", + "trap 'kill \"$port_forward_pid\" 2>/dev/null || true' EXIT\n", + "for _ in $(seq 1 30); do\n", + " curl -fsS http://127.0.0.1:18080/v2/health/ready >/dev/null 2>&1 && break\n", + " sleep 2\n", + "done\n", + "curl -fsS -X POST \\\n", + " http://127.0.0.1:18080/v2/models/feast-online-model/infer \\\n", + " -H 'Content-Type: application/json' \\\n", + " -d '{\"inputs\":[{\"name\":\"driver_id\",\"shape\":[2],\"datatype\":\"INT64\",\"data\":[1,2]}]}'\n", + "echo\n" ] } ], diff --git a/e2e/cases/c16_feast_offline_online.sh b/e2e/cases/c16_feast_offline_online.sh index 5c147eda..8bf81028 100755 --- a/e2e/cases/c16_feast_offline_online.sh +++ b/e2e/cases/c16_feast_offline_online.sh @@ -3,6 +3,9 @@ set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" source "$HERE/../lib.sh" +ASSET_DIR="${FEAST_ASSET_DIR:-$HERE/../../docs/en/train/guides/assets/feast-offline-to-online-inference}" +test -f "$ASSET_DIR/batch.py" +test -f "$ASSET_DIR/server.py" require_env FEAST_NAMESPACE "namespace for Feast e2e resources" require_env FEAST_IMAGE "Feast model-server image from the cluster registry" @@ -99,186 +102,9 @@ CLIENT="$(feast_kc -n "$NS" get featurestore "$FS_NAME" -o jsonpath='{.status.cl ONLINE_TLS="feast-$FS_NAME-online-tls" REGISTRY_TLS="feast-$FS_NAME-registry-tls" -cat >"$TMP/batch.py" <<'PY' -import copy -import json -import os -import shutil -import subprocess -from pathlib import Path -from urllib.parse import urlparse - -import numpy as np -import pandas as pd -import yaml -from feast import FeatureStore -from pyspark.sql import SparkSession, functions as F - -project = os.environ["FEAST_PROJECT"] -bucket = os.environ["S3_BUCKET"] -dataset_key = os.environ["S3_DATASET_KEY"].strip("/") -dataset_uri = f"s3a://{bucket}/{dataset_key}" -region = os.environ["AWS_DEFAULT_REGION"] -repo = Path("/tmp/feast-repo") -model_repo = Path("/mnt/models/repo") -repo.mkdir(parents=True, exist_ok=True) -model_repo.mkdir(parents=True, exist_ok=True) - -endpoint = urlparse(os.environ["S3_ENDPOINT_URL"]) -endpoint_host = endpoint.netloc or endpoint.path -ssl_enabled = str(endpoint.scheme == "https").lower() -spark = ( - SparkSession.builder - .appName("feast-offline-online-e2e") - .config("spark.hadoop.fs.s3a.endpoint", endpoint_host) - .config("spark.hadoop.fs.s3a.endpoint.region", region) - .config("spark.hadoop.fs.s3a.path.style.access", "true") - .config("spark.hadoop.fs.s3a.connection.ssl.enabled", ssl_enabled) - .getOrCreate() -) - -events = ( - spark.range(240) - .withColumn("driver_id", (F.col("id") % 12 + 1).cast("long")) - .withColumn("event_timestamp", F.timestamp_seconds(F.lit(1767225600) + F.col("id") * 3600)) - .withColumn("created", F.col("event_timestamp") + F.expr("INTERVAL 1 MINUTE")) - .withColumn("conv_rate", (F.lit(0.25) + F.lit(0.55) * F.rand(7)).cast("float")) - .withColumn("acc_rate", (F.lit(0.50) + F.lit(0.45) * F.rand(11)).cast("float")) - .withColumn("avg_daily_trips", F.floor(F.lit(2) + F.lit(18) * F.rand(13)).cast("long")) - .withColumn( - "label", - ((F.col("conv_rate") * 2 + F.col("acc_rate") + F.col("avg_daily_trips") / 20) > 1.8).cast("long"), - ) - .drop("id") -) -events.repartition(4, "driver_id").write.mode("overwrite").partitionBy("driver_id").parquet(dataset_uri) - -client_config = yaml.safe_load(Path("/etc/feast/feature_store.yaml").read_text()) -batch_config = copy.deepcopy(client_config) -batch_config["offline_store"] = { - "type": "spark", - "spark_conf": { - "spark.sql.session.timeZone": "UTC", - "spark.hadoop.fs.s3a.endpoint": endpoint_host, - "spark.hadoop.fs.s3a.endpoint.region": region, - "spark.hadoop.fs.s3a.path.style.access": "true", - "spark.hadoop.fs.s3a.connection.ssl.enabled": ssl_enabled, - }, -} -(repo / "feature_store.yaml").write_text(yaml.safe_dump(batch_config, sort_keys=False)) -(repo / "features.py").write_text(f'''from datetime import timedelta -from feast import Entity, FeatureService, FeatureView, Field -from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import SparkSource -from feast.types import Float32, Int64 -from feast.value_type import ValueType - -driver = Entity(name="driver", join_keys=["driver_id"], value_type=ValueType.INT64) -source = SparkSource( - name="driver_stats_source", - path="{dataset_uri}", - file_format="parquet", - timestamp_field="event_timestamp", - created_timestamp_column="created", -) -view = FeatureView( - name="driver_hourly_stats", - entities=[driver], - ttl=timedelta(days=365), - schema=[ - Field(name="conv_rate", dtype=Float32), - Field(name="acc_rate", dtype=Float32), - Field(name="avg_daily_trips", dtype=Int64), - ], - online=True, - source=source, -) -driver_activity_v1 = FeatureService(name="driver_activity_v1", features=[view]) -''') - -subprocess.run(["feast", "--chdir", str(repo), "apply"], check=True) -store = FeatureStore(repo_path=str(repo)) -entity_df = events.select("driver_id", "event_timestamp", "label").toPandas() -training = store.get_historical_features( - entity_df=entity_df, - features=[ - "driver_hourly_stats:conv_rate", - "driver_hourly_stats:acc_rate", - "driver_hourly_stats:avg_daily_trips", - ], -).to_df().dropna() -columns = ["conv_rate", "acc_rate", "avg_daily_trips"] -x = training[columns].to_numpy(dtype="float64") -y = training["label"].to_numpy(dtype="float64") -weights = np.linalg.pinv(np.column_stack([np.ones(len(x)), x])) @ y -np.savez("/mnt/models/model.npz", weights=weights, feature_columns=np.array(columns)) - -end_date = events.agg(F.max("event_timestamp")).first()[0] + pd.Timedelta(hours=1) -store.materialize_incremental(end_date) -online = store.get_online_features( - features=store.get_feature_service("driver_activity_v1"), - entity_rows=[{"driver_id": 1}, {"driver_id": 2}], -).to_df() -if len(online) != 2 or online[columns].isna().any().any(): - raise RuntimeError(f"online feature verification failed: {online}") - -serving_config = copy.deepcopy(client_config) -serving_config.pop("offline_store", None) -(model_repo / "feature_store.yaml").write_text(yaml.safe_dump(serving_config, sort_keys=False)) -shutil.copy("/opt/feast-batch/server.py", model_repo / "server.py") -print(json.dumps({"historical_rows": len(training), "online_rows": len(online)})) - -spark.stop() -PY - -cat >"$TMP/server.py" <<'PY' -import os -import numpy as np -import uvicorn -from fastapi import Body, FastAPI -from feast import FeatureStore - -name = os.getenv("MODEL_NAME", "feast-online-model") -weights = np.load("/mnt/models/model.npz")["weights"] -store = FeatureStore(repo_path="/mnt/models/repo") -service = store.get_feature_service("driver_activity_v1") -app = FastAPI() - -@app.get("/v2/health/ready") -@app.get("/v2/health/live") -def health(): - return {"ready": True} - -@app.get("/v2/models/{model_name}") -@app.get("/v2/models/{model_name}/ready") -def ready(model_name): - return {"name": model_name, "ready": model_name == name} - -@app.post("/v2/models/{model_name}/infer") -def infer(model_name, payload: dict = Body(...)): - ids = next(item for item in payload["inputs"] if item["name"] == "driver_id")["data"] - values = store.get_online_features( - features=service, - entity_rows=[{"driver_id": int(value)} for value in ids], - ).to_dict() - def column(column_name): - if column_name in values: - return values[column_name] - return values[next(key for key in values if key.endswith("__" + column_name))] - matrix = np.column_stack([ - np.ones(len(ids)), column("conv_rate"), column("acc_rate"), column("avg_daily_trips") - ]) - predictions = (matrix @ weights).astype("float32").tolist() - return { - "model_name": model_name, - "outputs": [{"name": "prediction", "shape": [len(ids)], "datatype": "FP32", "data": predictions}], - } - -if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=8080) -PY - feast_kc -n "$NS" create configmap "$CM" \ - --from-file=batch.py="$TMP/batch.py" --from-file=server.py="$TMP/server.py" \ + --from-file=batch.py="$ASSET_DIR/batch.py" \ + --from-file=server.py="$ASSET_DIR/server.py" \ --dry-run=client -o yaml | feast_kc apply -f - >/dev/null cat < Date: Thu, 20 Aug 2026 15:03:02 +0800 Subject: [PATCH 7/7] docs: document Feast notebook asset checkout --- .../guides/feast-offline-to-online-inference.ipynb | 12 +++++++++++- docs/en/train/guides/index.mdx | 12 ++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/en/train/guides/feast-offline-to-online-inference.ipynb b/docs/en/train/guides/feast-offline-to-online-inference.ipynb index 378953d9..66990c5b 100644 --- a/docs/en/train/guides/feast-offline-to-online-inference.ipynb +++ b/docs/en/train/guides/feast-offline-to-online-inference.ipynb @@ -15,7 +15,17 @@ "\n", "Object storage is the durable, versionable offline data layer; operator-managed Spark provides elastic batch compute; PostgreSQL stores Feast metadata rather than feature history; and Redis serves low-latency online lookups. Feast classifies its Spark offline store as a contributed integration without full test coverage, so qualify it against your scale and upgrade requirements or use a fully supported warehouse while retaining the same S3-and-Spark data pipeline.\n", "\n", - "The reusable source code and Kubernetes manifests are in `assets/feast-offline-to-online-inference/`. The command cells below assume that path is relative to the notebook working directory. Set `FEAST_ASSET_DIR` when your Workbench starts in a different directory.\n", + "The reusable source code and Kubernetes manifests are in `assets/feast-offline-to-online-inference/`. Downloading only this `.ipynb` will not include those required files. From a Workbench terminal, use a sparse checkout to obtain the notebook and asset bundle together:\n", + "\n", + "```bash\n", + "git clone --depth 1 --filter=blob:none --sparse https://github.com/alauda/aml-docs.git\n", + "git -C aml-docs sparse-checkout set --no-cone \\\n", + " /docs/en/train/guides/feast-offline-to-online-inference.ipynb \\\n", + " /docs/en/train/guides/assets/feast-offline-to-online-inference/\n", + "cd aml-docs/docs/en/train/guides\n", + "```\n", + "\n", + "Start the notebook from that `guides` directory so the default asset path resolves. Set `FEAST_ASSET_DIR` only when your Workbench uses a different working directory.\n", "\n", "Prerequisites:\n", "\n", diff --git a/docs/en/train/guides/index.mdx b/docs/en/train/guides/index.mdx index 0ff1abd7..93664a44 100644 --- a/docs/en/train/guides/index.mdx +++ b/docs/en/train/guides/index.mdx @@ -26,3 +26,15 @@ End-to-end recipes for training and fine-tuning models on Alauda AI. | Train tabular or time-series models with reusable AutoGluon assets | Managed KFP pipelines or composable components | [Use Reusable Kubeflow Pipeline Components](../../develop/pipelines/reusable-pipeline-components.mdx) | | Use KFP caching, parallel loops, and persistent typed artifacts | KFP 2.16.1 execution mechanisms | [Kubeflow Pipelines Execution and Storage Behavior](../../develop/pipelines/kfp-execution-and-storage.mdx) | | Daily fine-tune → evaluate → compare loop with MLflow + TrustyAI | KFP Recurring Run + MLflow Model Registry + `LMEvalJob` | [Daily Fine-Tuning Pipeline with MLflow and TrustyAI](./fine-tuning-pipeline-with-mlflow-trustyai.mdx) | + +:::note +The Feast notebook loads source code and manifests from its adjacent `assets/` directory. Clone the notebook and assets together instead of downloading the `.ipynb` alone: + +```bash +git clone --depth 1 --filter=blob:none --sparse https://github.com/alauda/aml-docs.git +git -C aml-docs sparse-checkout set --no-cone \ + /docs/en/train/guides/feast-offline-to-online-inference.ipynb \ + /docs/en/train/guides/assets/feast-offline-to-online-inference/ +cd aml-docs/docs/en/train/guides +``` +:::