{ "$schema": "https://json-schema.org/draft/2020-12/schema", "useCases": { "primary": [ { "id": "cloud-migration", "title": "Cloud Data Migration", "category": "Migration", "description": "Migrate on-premises database workloads to cloud platforms (AWS, Azure, GCP) with minimal downtime and maximum speed.", "problem": "Traditional migration tools are slow, complex, and require extensive infrastructure setup.", "solution": "FastBCP exports directly from source databases to cloud storage (S3, ADLS, GCS) in optimized formats like Parquet, achieving 30× faster migration times.", "benefits": [ "Minimal migration window", "Direct cloud upload eliminates staging", "Reduced cloud egress costs", "Parallel processing maximizes throughput" ], "example": { "scenario": "Oracle on-premises to Azure Data Lake", "dataVolume": "2TB production database", "fastBCPTime": "~2.5 hours", "traditionalTime": "~60 hours", "savings": "95% time reduction" }, "commandExample": "FastBCP --connectiontype oraodp --server oracle-prod:1521/PRODDB --user app_user --password *** --query \"SELECT * FROM sales_data\" --directory abfss://prodlake.dfs.core.windows.net/raw/ --fileoutput sales_data.parquet --parallelmethod Rowid --paralleldegree 16", "targetPersonas": ["Cloud Architects", "Migration Specialists", "Database Administrators"] }, { "id": "data-lake-ingestion", "title": "Data Lake & Data Warehouse Ingestion", "category": "Analytics", "description": "Extract data from operational databases into data lakes and cloud data warehouses for analytics and BI.", "problem": "ETL tools are expensive, complex to configure, and slow for large-scale data extraction.", "solution": "FastBCP provides high-speed extraction directly to Parquet format, optimized for analytics platforms like Snowflake, Databricks, and BigQuery.", "benefits": [ "Native Parquet support reduces conversion overhead", "Faster data availability for analytics", "Lower cost than full ETL platforms", "Simple command-line integration with orchestration tools" ], "example": { "scenario": "SQL Server OLTP to Snowflake via S3", "dataVolume": "500GB daily extract", "frequency": "Daily incremental load", "fastBCPTime": "~40 minutes", "traditionalTime": "~8 hours" }, "commandExample": "FastBCP --connectiontype mssql --server sqlprod --database Orders --trusted --query \"SELECT * FROM orders WHERE order_date >= DATEADD(day, -1, GETDATE())\" --directory s3://analytics-lake/snowflake-stage/ --fileoutput orders_incremental.parquet --parallelmethod Ntile --distributekeycolumn order_id --paralleldegree 12", "targetPersonas": ["Data Engineers", "Analytics Engineers", "BI Developers"] }, { "id": "database-backup", "title": "High-Speed Database Backup & Archive", "category": "Backup & Recovery", "description": "Create fast, compressed backups of database tables for disaster recovery and compliance archiving.", "problem": "Native backup tools don't support selective table export or cloud-native formats.", "solution": "FastBCP exports specific tables or query results in compressed Parquet format to local or cloud storage, providing flexible backup strategies.", "benefits": [ "Selective table backup", "Compressed storage reduces costs", "Cloud-native backup to S3/Azure/GCS", "Fast restore via Parquet import tools" ], "example": { "scenario": "PostgreSQL critical tables to Azure Blob", "dataVolume": "200GB across 50 tables", "fastBCPTime": "~20 minutes", "compressionRatio": "5:1 with Parquet" }, "commandExample": "FastBCP --connectiontype pgsql --server pg-prod --database app_db --user backup_user --password *** --fileinput backup-tables.sql --directory abs://backups.blob.core.windows.net/db-archives/ --fileoutput backup.parquet --parallelmethod Ctid --paralleldegree 8", "targetPersonas": ["Database Administrators", "DevOps Engineers", "Backup Administrators"] }, { "id": "cross-platform-sync", "title": "Cross-Platform Data Synchronization", "category": "Data Integration", "description": "Synchronize data between heterogeneous database platforms (Oracle to PostgreSQL, SQL Server to MySQL, etc.).", "problem": "Different database platforms have incompatible export/import formats and slow native tools.", "solution": "FastBCP extracts from any source database and exports to universal formats (CSV, Parquet, JSON) for import into target platforms.", "benefits": [ "Platform-agnostic data transfer", "No vendor lock-in", "Intermediate format suitable for multiple targets", "Schema evolution support via Parquet" ], "example": { "scenario": "Oracle legacy system to PostgreSQL cloud", "dataVolume": "1.5TB across 200 tables", "approach": "Extract to Parquet, load via psql/COPY", "fastBCPTime": "~90 minutes extraction" }, "commandExample": "FastBCP --connectiontype oraodp --server oracle-legacy:1521/LEGACYDB --user migrate_user --password *** --query \"SELECT * FROM customer_master\" --directory /staging/ --fileoutput customer_master.parquet --parallelmethod Rowid --paralleldegree 10", "targetPersonas": ["Data Migration Specialists", "Integration Engineers"] }, { "id": "etl-pipelines", "title": "ETL Pipeline Integration", "category": "Data Engineering", "description": "Integrate FastBCP into existing ETL workflows as a high-performance extraction component.", "problem": "Full ETL platforms are expensive and over-engineered for simple extraction needs.", "solution": "Use FastBCP as a lightweight, high-speed extraction tool orchestrated by Airflow, Prefect, or custom scripts.", "benefits": [ "Lower licensing costs than full ETL platforms", "Simple command-line interface for automation", "Parallel extraction accelerates pipelines", "Enterprise logging integrates with monitoring systems" ], "example": { "scenario": "Airflow DAG for nightly data warehouse refresh", "components": "FastBCP (extraction) → dbt (transformation) → Snowflake (load)", "improvement": "3× faster extraction vs. previous SSIS-based pipeline" }, "integrations": ["Apache Airflow", "Prefect", "Azure Data Factory", "AWS Step Functions", "Custom scripts (Python, PowerShell, Bash)"], "commandExample": "# Airflow BashOperator\nbash_command='FastBCP --connectiontype mssql --connectionstring \"${MSSQL_CONN}\" --fileinput /sql/extract.sql --directory s3://etl-stage/ --fileoutput extract.parquet --parallelmethod Physloc --paralleldegree 16'", "targetPersonas": ["Data Engineers", "DataOps Engineers", "Pipeline Developers"] }, { "id": "analytics-export", "title": "Analytical Reporting & BI Exports", "category": "Business Intelligence", "description": "Export query results and analytical datasets for business users, dashboards, and reporting tools.", "problem": "Business users need data in Excel or CSV format, but database exports are slow and manual.", "solution": "FastBCP enables fast, scheduled exports to Excel (XLSX) or CSV for distribution to business stakeholders.", "benefits": [ "Self-service data access for business users", "Automated report generation", "Excel-compatible outputs", "Fast query execution with parallel processing" ], "example": { "scenario": "Daily sales report to executive team", "output": "XLSX format with 1M rows", "delivery": "Emailed via automation script", "fastBCPTime": "~2 minutes" }, "commandExample": "FastBCP --connectiontype mssql --server analytics-db --database ReportsDB --trusted --query \"EXEC sp_daily_sales_report\" --directory /reports/ --fileoutput sales_$(date +%Y%m%d).xlsx --parallelmethod None", "targetPersonas": ["BI Analysts", "Data Analysts", "Report Developers"] }, { "id": "compliance-archival", "title": "Compliance & Regulatory Data Archival", "category": "Compliance", "description": "Archive historical data for regulatory compliance (GDPR, SOX, HIPAA) with audit trails.", "problem": "Regulatory requirements demand long-term data retention with immutable audit logs.", "solution": "FastBCP exports data to compressed, immutable Parquet archives in cloud storage with enterprise logging for audit trails.", "benefits": [ "Immutable cloud storage (S3 Glacier, Azure Archive)", "Compressed storage reduces costs", "Full audit trail via database logging", "Queryable archives via tools like Athena, Synapse" ], "example": { "scenario": "7-year retention of financial transactions", "dataVolume": "5TB/year", "storage": "S3 Glacier Deep Archive", "costSavings": "90% vs. keeping in production database" }, "commandExample": "FastBCP --connectiontype pgsql --server finance-db --database FinanceDB --user compliance_read --password *** --query \"SELECT * FROM transactions WHERE year = 2019\" --directory s3://compliance-archives/finance/2019/ --fileoutput transactions_2019.parquet --parallelmethod Ntile --distributekeycolumn txn_id --paralleldegree 8 --runid compliance-archive-2019", "targetPersonas": ["Compliance Officers", "Database Administrators", "Legal Teams"] }, { "id": "development-testing", "title": "Development & Testing Data Provisioning", "category": "DevOps", "description": "Create anonymized or sampled datasets for development, testing, and QA environments.", "problem": "Developers need production-like data but can't use full production datasets due to size and privacy.", "solution": "FastBCP extracts subsets or anonymized data from production databases quickly for dev/test environments.", "benefits": [ "Fast test data provisioning", "Sampling support via SQL queries", "Anonymization via SQL transformations", "Automated refresh of test environments" ], "example": { "scenario": "Weekly refresh of staging database", "approach": "Extract 10% sample + anonymize PII fields", "fastBCPTime": "~15 minutes", "consumption": "Import into local PostgreSQL container" }, "commandExample": "FastBCP --connectiontype pgsql --server prod-db --database AppDB --user read_only --password *** --query \"SELECT id, CONCAT('user_', id) AS username, order_date, amount FROM orders TABLESAMPLE BERNOULLI(10)\" --directory /testdata/ --fileoutput orders.csv --delimiter ',' --quotes --parallelmethod None", "targetPersonas": ["Developers", "QA Engineers", "DevOps Engineers"] } ], "industrySpecific": [ { "industry": "Financial Services", "useCases": ["Real-time risk data extraction", "Regulatory reporting", "Transaction archival"], "requirements": ["High security", "Audit trails", "Compliance"], "fastBCPFit": "Enterprise security, database logging, immutable cloud archives" }, { "industry": "Healthcare", "useCases": ["EHR data migration", "Claims data warehouse feeds", "HIPAA-compliant archival"], "requirements": ["Data privacy", "HIPAA compliance", "Large dataset handling"], "fastBCPFit": "Password obfuscation, secure cloud upload, streaming for large medical imaging metadata" }, { "industry": "Retail & E-commerce", "useCases": ["Customer data warehouse", "Inventory synchronization", "Sales analytics"], "requirements": ["High performance", "Real-time data availability", "Multi-region"], "fastBCPFit": "Parallel export reduces batch windows, direct cloud upload for global analytics" }, { "industry": "Manufacturing", "useCases": ["IoT sensor data archival", "Supply chain analytics", "ERP data extraction"], "requirements": ["High volume", "Time-series data", "Integration with BI tools"], "fastBCPFit": "Streaming architecture handles sensor data volumes, Parquet optimized for time-series" }, { "industry": "Telecommunications", "useCases": ["CDR (Call Detail Record) export", "Network analytics", "Customer data lakes"], "requirements": ["Massive scale", "Real-time processing", "Cost efficiency"], "fastBCPFit": "30× performance improvement on billion-row CDR tables, low memory footprint" } ], "summary": { "totalPrimaryUseCases": 8, "totalIndustries": 5, "commonThemes": ["Speed", "Cloud migration", "Cost reduction", "Simplicity", "Compliance"], "topUseCases": ["cloud-migration", "data-lake-ingestion", "etl-pipelines"] } }, "metadata": { "lastUpdated": "2026-02-19", "purpose": "Use case library for AI agents", "schemaVersion": "1.0.0" } }