Import data files

{
  "url": string,             // Required only for URL ingestion. The remote file location (HTTP or S3).
  "parquet_config": {        // Required. Defines how to ingest the Parquet file.
    "assetName": string,     // Required. Name of the Asset to attach this data to.
    "runName": string,       // Optional. Name of the Run to create for this data.
    "runId": string,         // Optional. ID of the Run to add this data to.
                            // If set, "runName" is ignored.
    "flatDataset": {         // Required. ParquetFlatDatasetConfig object.
      "timeColumn": {
        "path": string,           // Required. Path to the time column.
        "format": string,         // Required. See the "Time formats" section for options.
        "relativeStartTime": string // Optional. RFC3339 format. Used for relative time formats.
      },
      "dataColumns": [
        {
          "path": string,         // Required. Path to the data column. See the "Nested columns"
                                  // section for how to specify the path for nested columns.
          "channelConfig": {
            "name": string,        // Required. Name of the Channel.
            "dataType": string,    // Required. See the "Data types" section for options.
            "units": string,       // Optional. Channel units (defaults to empty).
            "description": string, // Optional. Channel description.
            "enumTypes": [         // Optional. Only valid if dataType is ENUM.
              {
                "key": number,     // Raw enum value.
                "name": string     // Display value for the enum.
              }
            ],
            "bitFieldElements": [  // Optional. Only valid if dataType is BIT_FIELD.
              {
                "index": number,   // Starting index of the bit field.
                "name": string,    // Name of the bit field element.
                "bitCount": number // Number of bits in the element.
              }
            ]
          }
        }
      ]
    },
    "footerOffset": number,  // Required. Byte position where the Parquet footer starts.
    "footerLength": number,  // Required. Length of the Parquet footer in bytes.
    "complexTypesImportMode": string // Optional. See the "Complex types" section for options. 
  }
}

# The following example shows how to extract the footer bytes from a Parquet file and submit
# them to the `detect-config` endpoint to generate a configuration.
import base64
import gzip
import json
import os
import struct
import requests

# Calculate Parquet footer offset and length
file_path = "sample_asset.parquet"
with open(file_path, "rb") as f:
    # See the "Parquet footer" section for how to calculate this.
    footer_len, footer_offset = extract_footer_information(file_path)
    f.seek(-(footer_len + 8), os.SEEK_END)
    footer_bytes = f.read(footer_len)

# Encode footer for detect-config
encoded_data = base64.b64encode(footer_bytes).decode("utf-8")
request_data = json.dumps({
    "data": encoded_data,
    "type": "DATA_TYPE_KEY_PARQUET_FLATDATASET",
})

# POST footer to detect-config endpoint
detect_config_url = f"{sift_uri}/api/v0/data-imports:detect-config"
headers = {"authorization": f"Bearer {apikey}"}
response = requests.post(detect_config_url, data=request_data, headers=headers)
response.raise_for_status()
config_info = response.json()
parquet_config = config_info["parquetConfig"]

# Update detected config with required fields
parquet_config["assetName"] = asset_name
parquet_config["flatDataset"]["timeColumn"]["path"] = "timestamp"
parquet_config["flatDataset"]["timeColumn"]["format"] = "TIME_FORMAT_ABSOLUTE_DATETIME"
parquet_config["footerOffset"] = footer_offset
parquet_config["footerLength"] = footer_len

# Use the updated parquet_config in your upload request to complete ingestion
# ....