DataStore Factory Methods

July 7, 2026 · View on GitHub

This document describes the convenient factory methods for creating DataStore instances from different data sources.

Overview

DataStore provides two ways to create instances:

Generic constructor: DataStore(source_type, **params)
Static factory methods: DataStore.from_xxx(...) (recommended for IDE auto-completion)

Factory Methods

File Operations

`DataStore.from_file()`

Create DataStore from local files with automatic format detection.

from datastore import DataStore

# Auto-detect format from extension
ds = DataStore.from_file("data.parquet")

# Explicit format
ds = DataStore.from_file("data.csv", format="CSV")

# With structure
ds = DataStore.from_file("data.csv", 
                         format="CSV",
                         structure="id UInt32, name String")

# Glob patterns
ds = DataStore.from_file("logs/*.csv", format="CSV")

Signature:

@classmethod
def from_file(cls, path: str, format: str = None, structure: str = None, 
              compression: str = None, **kwargs) -> 'DataStore'

Cloud Storage

`DataStore.from_s3()`

Create DataStore from Amazon S3 with automatic format detection.

# Public bucket with auto-detection
ds = DataStore.from_s3("s3://bucket/data.parquet", nosign=True)

# With credentials
ds = DataStore.from_s3(
    "s3://bucket/data/*.parquet",
    access_key_id="YOUR_KEY",
    secret_access_key="YOUR_SECRET"
)

# Explicit format
ds = DataStore.from_s3(
    "s3://bucket/data.csv",
    access_key_id="KEY",
    secret_access_key="SECRET",
    format="CSV"
)

Signature:

@classmethod
def from_s3(cls, url: str, access_key_id: str = None, 
            secret_access_key: str = None, format: str = None, 
            nosign: bool = False, **kwargs) -> 'DataStore'

`DataStore.from_hdfs()`

Create DataStore from HDFS with automatic format detection.

# Auto-detect format
ds = DataStore.from_hdfs("hdfs://namenode:9000/data/*.parquet")

# Explicit format
ds = DataStore.from_hdfs(
    "hdfs://namenode:9000/data/*.csv",
    format="CSV"
)

Signature:

@classmethod
def from_hdfs(cls, uri: str, format: str = None, 
              structure: str = None, **kwargs) -> 'DataStore'

Database Connections

Database factory methods support two modes:

Table-level: Specify database and table to bind to a specific table
Connection-level: Omit database/table to explore and query dynamically

`DataStore.from_mysql()`

Create DataStore from MySQL database.

# Table-level: bind to specific table
ds = DataStore.from_mysql(
    host="localhost:3306",
    database="mydb",
    table="users",
    user="root",
    password="password"
)
result = ds.select("*").filter(ds.age > 18).execute()

# Connection-level: explore and query dynamically
ds = DataStore.from_mysql(
    host="localhost:3306",
    user="root",
    password="password"
)
ds.databases()                    # List all databases
ds.tables("mydb")                 # List tables in database
ds.describe("mydb", "users")      # Get table schema
result = ds.sql("SELECT * FROM mydb.users WHERE age > 18")

Signature:

@classmethod
def from_mysql(cls, host: str, user: str, password: str = "",
               database: str = None, table: str = None, **kwargs) -> 'DataStore'

`DataStore.from_postgresql()`

Create DataStore from PostgreSQL database.

# Table-level
ds = DataStore.from_postgresql(
    host="localhost:5432",
    database="mydb",
    table="public.users",
    user="postgres",
    password="password"
)
result = ds.select("*").execute()

# Connection-level
ds = DataStore.from_postgresql(
    host="localhost:5432",
    user="postgres",
    password="password"
)
ds.databases()
ds.tables("mydb")
users = ds["mydb.users"]  # Select table, returns new DataStore

Signature:

@classmethod
def from_postgresql(cls, host: str, user: str, password: str = "",
                    database: str = None, table: str = None, **kwargs) -> 'DataStore'

`DataStore.from_clickhouse()`

Create DataStore from remote ClickHouse server.

# Table-level: bind to specific table
ds = DataStore.from_clickhouse(
    host="localhost:9000",
    database="default",
    table="events",
    user="default"
)
result = ds.select("*").filter(ds.date >= '2024-01-01').execute()

# Connection-level: explore and query dynamically
ds = DataStore.from_clickhouse(
    host="analytics.company.com:9440",
    user="analyst",
    password="secret",
    secure=True
)

# Metadata discovery
ds.databases()                      # ['default', 'production', 'staging']
ds.tables("production")             # ['users', 'orders', 'events']
ds.describe("production", "users")  # Table schema (columns, types)

# Set default database for shorter queries
ds.use("production")
result = ds.sql("SELECT * FROM users WHERE age > 25")

# Or select table directly (returns new DataStore)
users = ds["production.users"]
result = users.filter(users['age'] > 25).head(10)

Signature:

@classmethod
def from_clickhouse(cls, host: str, user: str = "default", password: str = "",
                    database: str = None, table: str = None,
                    secure: bool = False, **kwargs) -> 'DataStore'

Connection-Level Methods

When creating a connection-level DataStore (without specifying table), these methods are available:

Method	Description
`ds.databases()`	List all databases
`ds.tables(database)`	List tables in a database
`ds.describe(database, table)`	Get table schema
`ds.sql(query)`	Execute SQL query
`ds.use(database)` / `ds.use(database, table)`	Set default context
`ds["db.table"]`	Select table (returns new DataStore)

Password masking: Sensitive fields are automatically masked in repr():

print(ds)
# DataStore(source='clickhouse', host='analytics.company.com:9440', 
#           user='analyst', password='***', secure=True)

`DataStore.from_mongodb()`

Create DataStore from MongoDB (read-only).

ds = DataStore.from_mongodb(
    host="localhost:27017",
    database="mydb",
    collection="users",
    user="admin",
    password="password"
)

# MongoDB is read-only
result = ds.select("*").limit(100).execute()

Signature:

@classmethod
def from_mongodb(cls, host: str, database: str, collection: str,
                 user: str, password: str = "", **kwargs) -> 'DataStore'

`DataStore.from_sqlite()`

Create DataStore from SQLite database (read-only).

ds = DataStore.from_sqlite(
    database_path="/path/to/database.db",
    table="users"
)

result = ds.select("*").filter(ds.age > 18).execute()

Signature:

@classmethod
def from_sqlite(cls, database_path: str, table: str, **kwargs) -> 'DataStore'

HTTP/URL Data

`DataStore.from_url()`

Create DataStore from HTTP/HTTPS URL.

# Basic URL
ds = DataStore.from_url(
    url="https://example.com/data.json",
    format="JSONEachRow"
)

# With custom headers
ds = DataStore.from_url(
    url="https://api.example.com/data",
    format="JSONEachRow",
    headers=[
        "Authorization: Bearer TOKEN",
        "Accept: application/json"
    ]
)

Signature:

@classmethod
def from_url(cls, url: str, format: str, structure: str = None, 
             headers: List[str] = None, **kwargs) -> 'DataStore'

Data Lake Formats

`DataStore.from_iceberg()`

Create DataStore from Apache Iceberg table (read-only).

ds = DataStore.from_iceberg(
    url="s3://warehouse/my_table",
    access_key_id="YOUR_KEY",
    secret_access_key="YOUR_SECRET"
)

result = ds.select("*").filter(ds.year == 2024).execute()

Signature:

@classmethod
def from_iceberg(cls, url: str, access_key_id: str = None, 
                 secret_access_key: str = None, **kwargs) -> 'DataStore'

`DataStore.from_delta()`

Create DataStore from Delta Lake table (read-only).

ds = DataStore.from_delta(
    url="s3://bucket/delta_table",
    access_key_id="YOUR_KEY",
    secret_access_key="YOUR_SECRET"
)

result = ds.select("*").execute()

Signature:

@classmethod
def from_delta(cls, url: str, access_key_id: str = None,
               secret_access_key: str = None, **kwargs) -> 'DataStore'

Data Generation

`DataStore.from_numbers()`

Create DataStore that generates number sequences.

# Generate 0 to 99
ds = DataStore.from_numbers(100)

# Generate 10 to 19
ds = DataStore.from_numbers(10, start=10)

# Generate even numbers 0, 2, 4, ..., 18
ds = DataStore.from_numbers(10, start=0, step=2)

result = ds.select("*").execute()

Signature:

@classmethod
def from_numbers(cls, count: int, start: int = None, 
                 step: int = None, **kwargs) -> 'DataStore'

From pandas DataFrame

`DataStore.from_df()`

Create DataStore from an existing pandas DataFrame.

import pandas as pd
from datastore import DataStore

# Create a DataFrame
df = pd.DataFrame({
    'name': ['Alice', 'Bob', 'Charlie'],
    'age': [25, 30, 35],
    'city': ['NYC', 'LA', 'Chicago']
})

# Wrap with DataStore
ds = DataStore.from_df(df, name='users')

# Use DataStore features
result = ds.filter(ds.age > 26).to_df()

# Execute SQL on DataFrame
result = ds.sql('age > 28 ORDER BY name').to_df()

# Mix SQL and pandas operations
ds['age_group'] = ds.age // 10 * 10
result = ds.sql('age_group >= 30').to_df()

Signature:

@classmethod
def from_df(cls, df, name: str = None) -> 'DataStore'

Parameters:

df: pandas DataFrame to wrap
name: Optional name for the data source (appears in explain output)

`DataStore.from_dataframe()`

Alias for from_df() with identical functionality.

# Both are equivalent
ds = DataStore.from_df(df, name='users')
ds = DataStore.from_dataframe(df, name='users')

Signature:

@classmethod
def from_dataframe(cls, df, name: str = None) -> 'DataStore'

Comparison: Factory Methods vs Generic Constructor

Using Factory Methods (Recommended)

# ✓ IDE auto-completion shows available parameters
# ✓ Type hints for parameters
# ✓ Clear and discoverable API

ds = DataStore.from_s3(
    url="s3://bucket/data.parquet",
    access_key_id="KEY",
    secret_access_key="SECRET"
)

Using Generic Constructor

# ✗ Need to remember parameter names
# ✗ No IDE auto-completion for source-specific params

ds = DataStore(
    "s3",
    url="s3://bucket/data.parquet",
    access_key_id="KEY",
    secret_access_key="SECRET"
)

Both methods produce the same result, but factory methods provide better developer experience.

Format Auto-Detection

The following table functions support automatic format detection from file extensions:

file(): Local files
url(): HTTP(S) URLs
s3(): Amazon S3
azure(): Azure Blob Storage
gcs(): Google Cloud Storage
hdfs(): HDFS

Examples

# All these will auto-detect format:
ds = DataStore.from_file("data.parquet")      # → Parquet
ds = DataStore.from_file("data.csv")          # → CSV
ds = DataStore.from_file("data.json")         # → JSON
ds = DataStore.from_s3("s3://bucket/file.orc", nosign=True)  # → ORC
ds = DataStore.from_hdfs("hdfs://nn:9000/data.arrow")        # → Arrow

Complete Example

from datastore import DataStore

# Read from S3 with auto-detection
s3_data = DataStore.from_s3(
    "s3://my-bucket/events/*.parquet",
    access_key_id="KEY",
    secret_access_key="SECRET"
)

# Read from MySQL
mysql_data = DataStore.from_mysql(
    host="localhost:3306",
    database="mydb",
    table="users",
    user="root",
    password="pass"
)

# Join different sources
result = s3_data.join(
    mysql_data,
    left_on="user_id",
    right_on="id"
).select(
    "user_id",
    "event_type",
    "name",
    "email"
).filter(
    s3_data.event_date >= '2024-01-01'
).execute()

# Generate test data
numbers = DataStore.from_numbers(1000)
test_data = numbers.select(
    numbers.number.as_("id"),
    (numbers.number * 10).as_("value")
).execute()

Benefits

Better IDE Support: Auto-completion shows available parameters
Type Safety: Parameter types are documented and checked
Discoverability: Users can explore available data sources through DataStore.from_* methods
Cleaner Code: More readable and self-documenting
Format Auto-Detection: No need to specify format for common file types

Migration Guide

Old Style

ds = DataStore("file", path="data.csv", format="CSV")
ds = DataStore("s3", url="s3://bucket/data.parquet", format="Parquet", nosign=True)
ds = DataStore("mysql", host="localhost:3306", database="db", table="users", ...)

New Style (Recommended)

ds = DataStore.from_file("data.csv")  # Format auto-detected
ds = DataStore.from_s3("s3://bucket/data.parquet", nosign=True)  # Format auto-detected
ds = DataStore.from_mysql("localhost:3306", "db", "users", ...)

Both styles are supported and produce identical results.