From 4a255e8ca1eba423d9ac57072c01fe1dc08b1ee8 Mon Sep 17 00:00:00 2001 From: Timo Brockmann Date: Thu, 25 Jun 2026 15:28:21 +0200 Subject: [PATCH 1/2] Add storage examples for databases and s3 --- README.md | 3 + storage/README.md | 8 + storage/database-access/README.md | 38 ++++ storage/database-access/python/mariadb.ipynb | 163 +++++++++++++++++ .../database-access/python/postgresql.ipynb | 161 +++++++++++++++++ .../database-access/python/sqlserver.ipynb | 171 ++++++++++++++++++ storage/database-access/r/mariadb.R | 26 +++ storage/database-access/r/postgresql.R | 26 +++ storage/database-access/r/sqlserver.R | 29 +++ storage/s3/README.md | 25 +++ storage/s3/python/s3_access.ipynb | 82 +++++++++ storage/s3/r/s3_access.R | 30 +++ 12 files changed, 762 insertions(+) create mode 100644 storage/README.md create mode 100644 storage/database-access/README.md create mode 100644 storage/database-access/python/mariadb.ipynb create mode 100644 storage/database-access/python/postgresql.ipynb create mode 100644 storage/database-access/python/sqlserver.ipynb create mode 100644 storage/database-access/r/mariadb.R create mode 100644 storage/database-access/r/postgresql.R create mode 100644 storage/database-access/r/sqlserver.R create mode 100644 storage/s3/README.md create mode 100644 storage/s3/python/s3_access.ipynb create mode 100644 storage/s3/r/s3_access.R diff --git a/README.md b/README.md index 05c3307..9dbe52e 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ For full platform documentation, see [docs.prokube.ai](https://docs.prokube.ai/) ├── pipelines # Kubeflow Pipelines examples ├── rstudio # RStudio examples ├── serving # model serving examples (KServe, vLLM, shadow deployments) +├── storage # object storage and database access examples ``` ## Note about storage @@ -35,6 +36,8 @@ Many S3 libraries use environment variables for their configuration — those ar `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `S3_ENDPOINT`. They are likely already available in your environment. You can also ask your admin about them. +See `storage/s3` for S3 examples and `storage/database-access` for relational database examples in Python and R. + ## Platform compatibility Some examples require a minimum prokube platform version. If an example is not listed, no specific version requirement is known. diff --git a/storage/README.md b/storage/README.md new file mode 100644 index 0000000..9adfdb7 --- /dev/null +++ b/storage/README.md @@ -0,0 +1,8 @@ +# Storage Examples + +This directory contains examples for accessing storage services from prokube labs. + +- `s3`: read and write data with S3-compatible object storage. +- `database-access`: connect to PostgreSQL, MariaDB/MySQL, and SQL Server-compatible databases. + +The examples are available for Python notebooks and RStudio. They use the packages already provided by the pk notebook and pk RStudio images. diff --git a/storage/database-access/README.md b/storage/database-access/README.md new file mode 100644 index 0000000..8bec485 --- /dev/null +++ b/storage/database-access/README.md @@ -0,0 +1,38 @@ +# Database Access + +These examples show how to connect from the pk notebook and pk RStudio images to common relational databases. + +The pk notebook and pk RStudio images already include the required ODBC drivers and language packages. Each example uses the Iris dataset, creates a table named `iris_example`, writes the data, reads a few rows back, and closes the connection. + +## Examples + +- PostgreSQL: `python/postgresql.ipynb`, `r/postgresql.R` +- MariaDB/MySQL: `python/mariadb.ipynb`, `r/mariadb.R` +- Microsoft SQL Server: `python/sqlserver.ipynb`, `r/sqlserver.R` + +Edit only the connection values at the top of the example you want to run: + +- `host` +- `port` +- `database` +- `user` + +The examples prompt for the password. + +## Drivers + +The Python notebooks use these database drivers: + +- PostgreSQL: `psycopg2` +- MariaDB/MySQL: `pyodbc` with `MariaDB Unicode` +- Microsoft SQL Server: `pyodbc` with `FreeTDS` + +The R scripts use `DBI` and `odbc` with the same ODBC drivers. + +## Microsoft SQL Server + +For Microsoft SQL servers you first need to find out which port the server is using. Open a terminal in RStudio and run: + +```sh +tsql -L -H +``` diff --git a/storage/database-access/python/mariadb.ipynb b/storage/database-access/python/mariadb.ipynb new file mode 100644 index 0000000..c230932 --- /dev/null +++ b/storage/database-access/python/mariadb.ipynb @@ -0,0 +1,163 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "812eb55e", + "metadata": {}, + "source": [ + "# MariaDB/MySQL Access\n", + "\n", + "Write the Iris dataset to MariaDB or MySQL and read it back." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "db21f0c0", + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "\n", + "import pandas as pd\n", + "\n", + "from sklearn.datasets import load_iris" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90a6af60", + "metadata": {}, + "outputs": [], + "source": [ + "host = \"\"\n", + "port = 3306\n", + "database = \"\"\n", + "user = \"\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3100e7e2", + "metadata": {}, + "outputs": [], + "source": [ + "df = load_iris(as_frame=True)\n", + "df.frame.head()" + ] + }, + { + "cell_type": "markdown", + "id": "502ba2cb", + "metadata": {}, + "source": [ + "## SQLAlchemy \n", + "[SQLAlchemy](https://www.sqlalchemy.org/) is a Python library that provides a database abstraction layer so you can connect to different SQL databases.\n", + "It works well with `pandas.to_sql()` and `pandas.read_sql()` and can be used with different backends. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a328ba04", + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy import URL, create_engine\n", + "\n", + "engine = create_engine(\n", + " URL.create(\n", + " \"mysql+pyodbc\",\n", + " username=user,\n", + " password=getpass.getpass(\"Database password: \"),\n", + " host=host,\n", + " port=port,\n", + " database=database,\n", + " query={\"driver\": \"MariaDB Unicode\"},\n", + " )\n", + ")\n", + "\n", + "df.frame.to_sql(\"iris_example\", engine, if_exists=\"replace\", index=False)\n", + "\n", + "pd.read_sql(\"SELECT * FROM iris_example\", engine).head()" + ] + }, + { + "cell_type": "markdown", + "id": "d0ebb300", + "metadata": {}, + "source": [ + "## pyodbc cursor\n", + "\n", + "Direct `pyodbc` cursor calls are more verbose, but they show the underlying DB-API flow: open a connection, create a cursor, execute SQL, commit, fetch rows, and close the connection.\n", + "\n", + "Documentation: [pyodbc](https://github.com/mkleehammer/pyodbc/wiki)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f34c3be1", + "metadata": {}, + "outputs": [], + "source": [ + "import pyodbc\n", + "\n", + "conn = pyodbc.connect(\n", + " \"DRIVER={MariaDB Unicode};\"\n", + " f\"SERVER={host};\"\n", + " f\"PORT={port};\"\n", + " f\"DATABASE={database};\"\n", + " f\"UID={user};\"\n", + " f\"PWD={getpass.getpass('Database password: ')}\"\n", + ")\n", + "\n", + "try:\n", + " cursor = conn.cursor()\n", + " cursor.execute(\"DROP TABLE IF EXISTS iris_example_cursor\")\n", + " cursor.execute(\"\"\"\n", + " CREATE TABLE iris_example_cursor (\n", + " `sepal length (cm)` DOUBLE,\n", + " `sepal width (cm)` DOUBLE,\n", + " `petal length (cm)` DOUBLE,\n", + " `petal width (cm)` DOUBLE,\n", + " target INTEGER\n", + " )\n", + " \"\"\")\n", + " cursor.executemany(\"\"\"\n", + " INSERT INTO iris_example_cursor (\n", + " `sepal length (cm)`,\n", + " `sepal width (cm)`,\n", + " `petal length (cm)`,\n", + " `petal width (cm)`,\n", + " target\n", + " )\n", + " VALUES (?, ?, ?, ?, ?)\n", + " \"\"\", df.frame.itertuples(index=False, name=None))\n", + " conn.commit()\n", + "\n", + " cursor.execute(\"SELECT * FROM iris_example_cursor LIMIT 5\")\n", + " rows = cursor.fetchall()\n", + " columns = [column[0] for column in cursor.description]\n", + " display(pd.DataFrame.from_records(rows, columns=columns))\n", + "finally:\n", + " conn.close()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/storage/database-access/python/postgresql.ipynb b/storage/database-access/python/postgresql.ipynb new file mode 100644 index 0000000..8ea8761 --- /dev/null +++ b/storage/database-access/python/postgresql.ipynb @@ -0,0 +1,161 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f3000bce", + "metadata": {}, + "source": [ + "# PostgreSQL Access\n", + "\n", + "Write the Iris dataset to PostgreSQL and read it back." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c0b806c1", + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "\n", + "import pandas as pd\n", + "\n", + "from sklearn.datasets import load_iris" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca059a31", + "metadata": {}, + "outputs": [], + "source": [ + "host = \"\"\n", + "port = 5432\n", + "database = \"\"\n", + "user = \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d97f6e9", + "metadata": {}, + "outputs": [], + "source": [ + "df = load_iris(as_frame=True)\n", + "df.frame.head()" + ] + }, + { + "cell_type": "markdown", + "id": "5c1bc368", + "metadata": {}, + "source": [ + "## SQLAlchemy \n", + "[SQLAlchemy](https://www.sqlalchemy.org/) is a Python library that provides a database abstraction layer so you can connect to different SQL databases.\n", + "It works well with `pandas.to_sql()` and `pandas.read_sql()` and can be used with different backends. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f48e4486", + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy import URL, create_engine\n", + "\n", + "engine = create_engine(\n", + " URL.create(\n", + " \"postgresql+psycopg2\",\n", + " username=user,\n", + " password=getpass.getpass(\"Database password: \"),\n", + " host=host,\n", + " port=port,\n", + " database=database,\n", + " )\n", + ")\n", + "\n", + "df.frame.to_sql(\"iris_example\", engine, if_exists=\"replace\", index=False)\n", + "pd.read_sql(\"SELECT * FROM iris_example\", engine).head()" + ] + }, + { + "cell_type": "markdown", + "id": "5fa69e55", + "metadata": {}, + "source": [ + "## psycopg2 cursor\n", + "\n", + "Direct `psycopg2` cursor calls are more verbose, but they show the underlying DB-API flow: open a connection, create a cursor, execute SQL, commit, fetch rows, and close the connection.\n", + "\n", + "Documentation: [psycopg2](https://www.psycopg.org/docs/index.html)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "859cbf65", + "metadata": {}, + "outputs": [], + "source": [ + "import psycopg2\n", + "\n", + "conn = psycopg2.connect(\n", + " host=host,\n", + " port=port,\n", + " dbname=database,\n", + " user=user,\n", + " password=getpass.getpass(\"Database password: \"),\n", + ")\n", + "\n", + "try:\n", + " with conn.cursor() as cursor:\n", + " cursor.execute(\"DROP TABLE IF EXISTS iris_example_cursor\")\n", + " cursor.execute(\"\"\"\n", + " CREATE TABLE iris_example_cursor (\n", + " \"sepal length (cm)\" DOUBLE PRECISION,\n", + " \"sepal width (cm)\" DOUBLE PRECISION,\n", + " \"petal length (cm)\" DOUBLE PRECISION,\n", + " \"petal width (cm)\" DOUBLE PRECISION,\n", + " target INTEGER\n", + " )\n", + " \"\"\")\n", + " cursor.executemany(\"\"\"\n", + " INSERT INTO iris_example_cursor (\n", + " \"sepal length (cm)\",\n", + " \"sepal width (cm)\",\n", + " \"petal length (cm)\",\n", + " \"petal width (cm)\",\n", + " target\n", + " )\n", + " VALUES (%s, %s, %s, %s, %s)\n", + " \"\"\", df.frame.itertuples(index=False, name=None))\n", + " conn.commit()\n", + "\n", + " cursor.execute(\"SELECT * FROM iris_example_cursor LIMIT 5\")\n", + " rows = cursor.fetchall()\n", + " columns = [column.name for column in cursor.description]\n", + "\n", + " display(pd.DataFrame(rows, columns=columns))\n", + "finally:\n", + " conn.close()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/storage/database-access/python/sqlserver.ipynb b/storage/database-access/python/sqlserver.ipynb new file mode 100644 index 0000000..d9eae6e --- /dev/null +++ b/storage/database-access/python/sqlserver.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "fe5bb86e", + "metadata": {}, + "source": [ + "# Microsoft SQL Server Access\n", + "\n", + "Write the Iris dataset to Microsoft SQL Server through FreeTDS and read it back." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b494a5d7", + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "\n", + "import pandas as pd\n", + "\n", + "from sklearn.datasets import load_iris" + ] + }, + { + "cell_type": "markdown", + "id": "53c44dcc", + "metadata": {}, + "source": [ + "If you do not know the SQL Server port, open a terminal in RStudio and run `tsql -L -H `." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc56b6ab", + "metadata": {}, + "outputs": [], + "source": [ + "host = \"\"\n", + "port = 1433\n", + "database = \"\"\n", + "user = \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eb4a444e", + "metadata": {}, + "outputs": [], + "source": [ + "df = load_iris(as_frame=True)\n", + "df.frame.head()" + ] + }, + { + "cell_type": "markdown", + "id": "fc290b98", + "metadata": {}, + "source": [ + "## SQLAlchemy \n", + "[SQLAlchemy](https://www.sqlalchemy.org/) is a Python library that provides a database abstraction layer so you can connect to different SQL databases.\n", + "It works well with `pandas.to_sql()` and `pandas.read_sql()` and can be used with different backends. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cce8a9d0", + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy import URL, create_engine\n", + "\n", + "engine = create_engine(\n", + " URL.create(\n", + " \"mssql+pyodbc\",\n", + " username=user,\n", + " password=getpass.getpass(\"Database password: \"),\n", + " host=host,\n", + " port=port,\n", + " database=database,\n", + " query={\"driver\": \"FreeTDS\", \"TDS_Version\": \"7.4\"},\n", + " )\n", + ")\n", + "\n", + "df.frame.to_sql(\"iris_example\", engine, if_exists=\"replace\", index=False)\n", + "pd.read_sql(\"SELECT * FROM iris_example\", engine).head()" + ] + }, + { + "cell_type": "markdown", + "id": "f5919940", + "metadata": {}, + "source": [ + "## pyodbc cursor\n", + "\n", + "Direct `pyodbc` cursor calls are more verbose, but they show the underlying DB-API flow: open a connection, create a cursor, execute SQL, commit, fetch rows, and close the connection.\n", + "\n", + "Documentation: [pyodbc](https://github.com/mkleehammer/pyodbc/wiki)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d110276", + "metadata": {}, + "outputs": [], + "source": [ + "import pyodbc\n", + "\n", + "conn = pyodbc.connect(\n", + " \"DRIVER={FreeTDS};\"\n", + " f\"SERVER={host};\"\n", + " f\"PORT={port};\"\n", + " f\"DATABASE={database};\"\n", + " f\"UID={user};\"\n", + " f\"PWD={getpass.getpass('Database password: ')};\"\n", + " \"TDS_Version=7.4\"\n", + ")\n", + "\n", + "try:\n", + " cursor = conn.cursor()\n", + " cursor.execute(\"DROP TABLE IF EXISTS iris_example_cursor\")\n", + " cursor.execute(\"\"\"\n", + " CREATE TABLE iris_example_cursor (\n", + " [sepal length (cm)] FLOAT,\n", + " [sepal width (cm)] FLOAT,\n", + " [petal length (cm)] FLOAT,\n", + " [petal width (cm)] FLOAT,\n", + " target INTEGER\n", + " )\n", + " \"\"\")\n", + " cursor.executemany(\"\"\"\n", + " INSERT INTO iris_example_cursor (\n", + " [sepal length (cm)],\n", + " [sepal width (cm)],\n", + " [petal length (cm)],\n", + " [petal width (cm)],\n", + " target\n", + " )\n", + " VALUES (?, ?, ?, ?, ?)\n", + " \"\"\", df.frame.itertuples(index=False, name=None))\n", + " conn.commit()\n", + "\n", + " cursor.execute(\"SELECT TOP 5 * FROM iris_example_cursor\")\n", + " rows = cursor.fetchall()\n", + " columns = [column[0] for column in cursor.description]\n", + " display(pd.DataFrame.from_records(rows, columns=columns))\n", + "finally:\n", + " conn.close()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/storage/database-access/r/mariadb.R b/storage/database-access/r/mariadb.R new file mode 100644 index 0000000..390e994 --- /dev/null +++ b/storage/database-access/r/mariadb.R @@ -0,0 +1,26 @@ +library(DBI) +library(odbc) +library(rstudioapi) + +host <- "" +port <- 3306 +database <- "" +user <- "" +password <- askForPassword("Database password") + +con <- dbConnect( + odbc(), + Driver = "MariaDB Unicode", + Server = host, + Port = port, + Database = database, + UID = user, + PWD = password +) + +tryCatch({ + dbWriteTable(con, "iris_example", iris, overwrite = TRUE, row.names = FALSE) + print(head(dbReadTable(con, "iris_example"))) +}, finally = { + dbDisconnect(con) +}) diff --git a/storage/database-access/r/postgresql.R b/storage/database-access/r/postgresql.R new file mode 100644 index 0000000..b609805 --- /dev/null +++ b/storage/database-access/r/postgresql.R @@ -0,0 +1,26 @@ +library(DBI) +library(odbc) +library(rstudioapi) + +host <- "" +port <- 5432 +database <- "" +user <- "" +password <- askForPassword("Database password") + +con <- dbConnect( + odbc(), + Driver = "PostgreSQL Unicode", + Server = host, + Port = port, + Database = database, + UID = user, + PWD = password +) + +tryCatch({ + dbWriteTable(con, "iris_example", iris, overwrite = TRUE, row.names = FALSE) + print(head(dbReadTable(con, "iris_example"))) +}, finally = { + dbDisconnect(con) +}) diff --git a/storage/database-access/r/sqlserver.R b/storage/database-access/r/sqlserver.R new file mode 100644 index 0000000..db7c3d9 --- /dev/null +++ b/storage/database-access/r/sqlserver.R @@ -0,0 +1,29 @@ +library(DBI) +library(odbc) +library(rstudioapi) + +# If you do not know the port, run this in a terminal: +# tsql -L -H +host <- "" +port <- 1433 +database <- "" +user <- "" +password <- askForPassword("Database password") + +con <- dbConnect( + odbc(), + Driver = "FreeTDS", + Server = host, + Port = port, + Database = database, + UID = user, + PWD = password, + TDS_Version = "7.4" +) + +tryCatch({ + dbWriteTable(con, "iris_example", iris, overwrite = TRUE, row.names = FALSE) + print(head(dbReadTable(con, "iris_example"))) +}, finally = { + dbDisconnect(con) +}) diff --git a/storage/s3/README.md b/storage/s3/README.md new file mode 100644 index 0000000..f3807b5 --- /dev/null +++ b/storage/s3/README.md @@ -0,0 +1,25 @@ +# S3 Object Storage Access + +These examples show how to write and read the Iris dataset from prokube.ai's integrated S3 object storage. +You only need to set the bucket name in the example. + +## Python + +Open `python/s3_access.ipynb` in a pk notebook. + +The notebook uses the platform configuration directly: + +```python +import s3fs + +s3 = s3fs.S3FileSystem() + +with s3.open("/", "rb") as f: + print(f.read()) +``` + +## R + +Run `r/s3_access.R` in pk RStudio. + +The R example uses `aws.s3`. It sets `use_https = FALSE` because this example is for the preconfigured internal MinIO endpoint, where cluster traffic is protected by the platform. Do not use `use_https = FALSE` for external S3 endpoints. diff --git a/storage/s3/python/s3_access.ipynb b/storage/s3/python/s3_access.ipynb new file mode 100644 index 0000000..8540ea8 --- /dev/null +++ b/storage/s3/python/s3_access.ipynb @@ -0,0 +1,82 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c749cac1", + "metadata": {}, + "source": [ + "# Access MinIO with Python\n", + "\n", + "Write the Iris dataset to MinIO and read it back." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6452253", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import s3fs\n", + "from sklearn.datasets import load_iris" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4ad02161", + "metadata": {}, + "outputs": [], + "source": [ + "bucket = \"\"\n", + "key = \"storage-examples/iris.csv\"\n", + "\n", + "if not bucket or bucket == \"\":\n", + " raise ValueError(\"Set bucket to your MinIO bucket name.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "437e8c92", + "metadata": {}, + "outputs": [], + "source": [ + "df = load_iris(as_frame=True)\n", + "df.frame.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8f872e53", + "metadata": {}, + "outputs": [], + "source": [ + "s3 = s3fs.S3FileSystem()\n", + "\n", + "with s3.open(f\"{bucket}/{key}\", \"w\") as f:\n", + " df.frame.to_csv(f, index=False)\n", + "\n", + "with s3.open(f\"{bucket}/{key}\", \"rb\") as f:\n", + " read_back = pd.read_csv(f)\n", + "\n", + "read_back.head()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/storage/s3/r/s3_access.R b/storage/s3/r/s3_access.R new file mode 100644 index 0000000..3936eb7 --- /dev/null +++ b/storage/s3/r/s3_access.R @@ -0,0 +1,30 @@ +library(aws.s3) + +bucket <- "" +if (!nzchar(bucket) || bucket == "") { + stop("Set bucket to your S3 bucket name.", call. = FALSE) +} + +key <- "storage-examples/iris.csv" + +csv_path <- tempfile(fileext = ".csv") +write.csv(iris, csv_path, row.names = FALSE) + +put_object( + file = csv_path, + object = key, + bucket = bucket, + use_https = FALSE, + region = "" +) + +obj <- get_object( + object = key, + bucket = bucket, + use_https = FALSE, + region = "" +) + +read_back <- read.csv(text = rawToChar(obj)) + +head(read_back) From f57edf7ec3d9a46f9451f4b3ddbe1239f94cbd78 Mon Sep 17 00:00:00 2001 From: Timo Brockmann Date: Tue, 14 Jul 2026 12:27:49 +0200 Subject: [PATCH 2/2] Add DuckDB example --- storage/database-access/python/duckdb.ipynb | 102 ++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 storage/database-access/python/duckdb.ipynb diff --git a/storage/database-access/python/duckdb.ipynb b/storage/database-access/python/duckdb.ipynb new file mode 100644 index 0000000..8672788 --- /dev/null +++ b/storage/database-access/python/duckdb.ipynb @@ -0,0 +1,102 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "250f9140", + "metadata": {}, + "source": [ + "## DuckDB\n", + "DuckDB can read files from S3-backed storage through its `httpfs` extension. In managed Labs, S3-compatible endpoint and credential environment variables are usually provided by the workspace configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b98f465", + "metadata": {}, + "outputs": [], + "source": [ + "# Install duckdb\n", + "!pip install duckdb" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ba83c11", + "metadata": {}, + "outputs": [], + "source": [ + "import duckdb\n", + "import os \n", + "\n", + "conn = duckdb.connect()\n", + "conn.execute(\"INSTALL httpfs\")\n", + "conn.execute(\"LOAD httpfs\")\n", + "conn.execute(\"SET s3_url_style='path'\")\n", + "conn.execute(f\"SET s3_endpoint='{os.environ['S3_ENDPOINT']}'\")\n", + "conn.execute(f\"SET s3_use_ssl = {'false' if os.environ['S3_USE_HTTPS'] == '0' else 'true'}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d990fd48", + "metadata": {}, + "source": [ + "To check whether DuckDB can discover credentials in the current environment, run `CALL load_aws_credentials()` after loading the `httpfs` extension." + ] + }, + { + "cell_type": "markdown", + "id": "8d7c6b5a", + "metadata": {}, + "source": [ + "## Iris dataset in S3 Parquet\n", + "\n", + "Write the Iris dataset to an S3-backed Parquet file and read it back with DuckDB." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f5e4d3c", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.datasets import load_iris\n", + "\n", + "bucket = \"\"\n", + "key = \"storage-examples/iris.parquet\"\n", + "\n", + "if not bucket or bucket == \"\":\n", + " raise ValueError(\"Set bucket to your S3 bucket name.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b1a0f9e", + "metadata": {}, + "outputs": [], + "source": [ + "iris = load_iris(as_frame=True).frame\n", + "s3_path = f\"s3://{bucket}/{key}\"\n", + "\n", + "conn.register(\"iris\", iris)\n", + "conn.execute(\n", + " f\"COPY iris TO '{s3_path}' (FORMAT PARQUET, OVERWRITE_OR_IGNORE TRUE)\"\n", + ")\n", + "\n", + "read_back = conn.sql(f\"SELECT * FROM read_parquet('{s3_path}')\").df()\n", + "read_back.head()" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}