Protocol Buffers
The following protocol buffers are used for protobuf ingestion.
ProtobufDescriptorService
syntax = "proto3";
package azimuth.protobuf_descriptors.v1;
import "google/api/annotations.proto";
import "google/api/field_behavior.proto";
import "protoc-gen-openapiv2/options/annotations.proto";
option go_package = "azimuth/gen/protos/go/azimuth/protobuf_descriptors/v1;protobufdescriptorsv1pb";
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = {
info: {title: "Protobuf Descriptor Service"}
};
service ProtobufDescriptorService {
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_tag) = {
description: "Service to programmatically interact with protobuf desciptors used for protobuf ingestion."
external_docs: {
url: "/ingestion/protobuf_ingestion";
description: "Read more about what protobuf ingestion.";
}
};
// Used to register a protobuf message to be ingested.
rpc AddProtobufDescriptor(AddProtobufDescriptorRequest) returns (AddProtobufDescriptorResponse) {
option (google.api.http) = {post: "/api/v1/protobuf-descriptors:add"};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
summary: "AddProtobufDescriptor"
description: "Used to register a protobuf message to be ingested."
};
}
// Delete protobuf descriptors of that match the provided `namespace` and `message_type_full_name`.
rpc DeleteProtobufDescriptors(DeleteProtobufDescriptorsRequest) returns (DeleteProtobufDescriptorsResponse) {
option (google.api.http) = {delete: "/api/v1/protobuf-descriptors"};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
summary: "DeleteProtobufDescriptors"
description: "Delete protobuf descriptors of that match the provided `namespace` and `message_type_full_name`."
};
}
// Retrieve protobuf descriptors using an optional filter.
rpc ListProtobufDescriptors(ListProtobufDescriptorsRequest) returns (ListProtobufDescriptorsResponse) {
option (google.api.http) = {get: "/api/v1/protobuf-descriptors"};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
summary: "ListProtobufDescriptors"
description: "Retrieve protobuf descriptors using an optional filter."
};
}
}
message DeleteProtobufDescriptorsRequest {
string message_type_full_name = 1 [deprecated = true];
string namespace = 2 [deprecated = true];
string organization_id = 3;
string protobuf_descriptor_id = 4;
}
message DeleteProtobufDescriptorsResponse {}
message AddProtobufDescriptorRequest {
ProtobufDescriptor protobuf_descriptor = 1;
string organization_id = 2;
}
message AddProtobufDescriptorResponse {
ProtobufDescriptor protobuf_descriptor = 1;
}
message ProtobufDescriptor {
string message_type_full_name = 1;
bytes file_descriptor_set = 2;
string proto_file_name = 3;
string namespace = 4;
string protobuf_descriptor_id = 5;
}
// The request for a call to `ProtobufDescriptorService_ListProtobufDescriptors` to retrieve protobuf descriptors.
message ListProtobufDescriptorsRequest {
// The ID of organization associated with the requested protobuf descriptors.
string organization_id = 1 [(google.api.field_behavior) = OPTIONAL];
// The maximum number of protobuf descriptors to return.
// The service may return fewer than this value.
// If unspecified, at most 50 protobuf descriptors will be returned.
// The maximum value is 1000; values above 1000 will be coerced to 1000.
uint32 page_size = 2 [(google.api.field_behavior) = OPTIONAL];
// A page token, received from a previous `ListProtobufDescriptors` call.
// Provide this to retrieve the subsequent page.
// When paginating, all other parameters provided to `ListProtobufDescriptors` must match
// the call that provided the page token.
string page_token = 3 [(google.api.field_behavior) = OPTIONAL];
// A [Common Expression Language (CEL)](https://github.com/google/cel-spec) filter string.
// Available fields to filter by are `protobuf_descriptor_id`, `proto_file_name`, `namespace`, and `message_type_full_name`.
// For further information about how to use CELs, please refer to [this guide](https://github.com/google/cel-spec/blob/master/doc/langdef.md#standard-definitions).
// For more information about the fields used for filtering, please refer to [this definition](/protocol-buffers/documentation#protobufdescriptor). Optional.
string filter = 4 [(google.api.field_behavior) = OPTIONAL];
}
// The response of a call to `ProtobufDescriptorService_ListProtobufDescriptors`.
message ListProtobufDescriptorsResponse {
repeated ProtobufDescriptor protobuf_descriptors = 1;
string next_page_token = 2;
}
DataIngestionService
syntax = "proto3";
import "google/protobuf/timestamp.proto";
import "google/protobuf/empty.proto";
import "google/api/field_behavior.proto";
option go_package = "azimuth/grpc/proto";
option java_package = "com.azimuth.dataingestion";
service DataIngestionService {
// Define the schema of the data you want to ingest by declaring a list of flows with the configuration of the channels
rpc CreateIngestionConfig (CreateIngestionConfigRequest) returns (CreateIngestionConfigResponse);
// This endpoint allows adding new flows to an existing ingestion config
rpc AmendIngestionConfig(AmendIngestionConfigRequest) returns (google.protobuf.Empty);
rpc GetIngestionConfig (GetIngestionConfigRequest) returns (GetIngestionConfigResponse);
// Use the ingestion config to send one flow of structured data. This endpoint will time out if it doesn't receive a message in 60 seconds.
rpc IngestData (stream IngestDataRequest) returns (google.protobuf.Empty);
rpc IngestProtobuf (stream IngestProtobufRequest) returns (google.protobuf.Empty);
rpc CreateRun (CreateRunRequest) returns (CreateRunResponse);
rpc UpdateRun (UpdateRunRequest) returns (UpdateRunResponse);
rpc StopRun (StopRunRequest) returns (StopRunResponse);
rpc CreateAutomaticRunAssociationForAssets (CreateAutomaticRunAssociationForAssetsRequest) returns (CreateAutomaticRunAssociationForAssetsResponse);
// Deprecated. Use `AddProtobufDescriptor` and `DeleteProtobufDescriptors` in `azimuth.protobuf_descriptors.v1`
rpc SaveProtobufDescriptor(SaveProtobufDescriptorRequest) returns (google.protobuf.Empty);
}
message CreateIngestionConfigRequest {
string asset_name = 1;
repeated FlowConfig flows = 2;
string organization_id = 3;
// client_key is a user-defined string you can supply to uniquely identify
// an ingestion config (and retrieve it via GetIngestionConfig).
// An error is returned if you try to create an ingestion config with a
// client_key that already exists.
string client_key = 4;
}
message CreateIngestionConfigResponse {
string ingestion_config_id = 1;
// The returned asset will be created if one didn't exist with a matching name.
string asset_id = 2;
}
message AmendIngestionConfigRequest {
string ingestion_config_id = 1;
// These flows must have unique names. If you try to send a flow with a name that already exists for an ingestion config, it will return an error.
repeated FlowConfig flows = 2;
// If client_key is not empty, it will amend the ingestion config with the matching client_key.
// It is an error to supply both client_key and ingestion_config_id.
string client_key = 3;
}
message GetIngestionConfigRequest {
string ingestion_config_id = 1;
// If client_key is supplied, it will return the ingestion config with the matching client_key.
// It is an error to supply both client_key and ingestion_config_id.
string client_key = 2;
}
message GetIngestionConfigResponse {
string asset_name = 1;
string asset_id = 2;
repeated FlowConfig flows = 3;
string ingestion_config_id = 4;
string client_key = 5;
}
// A flow lets a consumer configure groupings of channels that will be ingested together. A list of flows
// can be provided when creating the ingestion config, and then when ingesting data, you provide one flow per request.
message FlowConfig {
string name = 1;
repeated ChannelConfig channels = 2;
}
message ChannelConfig {
string name = 1;
string component = 2;
string units = 3;
string description = 4;
ChannelDataType data_type = 5;
repeated ChannelEnumType enum_types = 6;
repeated ChannelBitFieldElement bit_field_elements = 7;
}
enum ChannelDataType {
// Unused, see https://protobuf.dev/programming-guides/style/#enums
UNSPECIFIED = 0 [deprecated=true];
DOUBLE = 1;
STRING = 2;
ENUM = 3;
BIT_FIELD = 4;
BOOL = 5;
FLOAT = 6;
INT_32 = 7;
UINT_32 = 8;
INT_64 = 9;
UINT_64 = 10;
}
message ChannelEnumType {
string name = 1;
uint32 key = 2;
}
message ChannelBitFieldElement {
string name = 1;
// The index of this element's first bit in the logical bit field array.
int32 index = 2;
uint32 bit_count = 3;
}
message IngestDataRequest {
string ingestion_config_id = 1;
string flow = 2;
google.protobuf.Timestamp timestamp = 3;
repeated ChannelValue channel_values = 4;
// The run_id MUST be included if this data is part of a run.
string run_id = 5;
// By default, if this request contains any channel values that do not match
// the supplied ingestion config, the request is stored in an error queue and
// the stream continues to accept data. This ensures all data is saved, but
// only valid data is fully ingested. If this is set to `true`, any validation
// errors end the stream and return the error to the client.
bool end_stream_on_validation_error = 6;
string organization_id = 7;
}
message ChannelValue {
oneof type {
string string = 1;
double double = 2;
float float = 3;
bool bool = 4;
int32 int32 = 5;
uint32 uint32 = 6;
int64 int64 = 7;
uint64 uint64 = 8;
bytes bit_field = 9;
uint32 enum = 10;
// If there's not a new data point for a channel at the given timestamp, pass empty to skip it
google.protobuf.Empty empty = 11;
}
}
message RunAsset {
// Runs are automatically associated with assets, based on the data that's ingested with the run_id.
option deprecated = true;
string asset_id = 1;
optional string ingestion_config_id = 2;
}
message Run {
string run_id = 1;
// Runs are automatically associated with assets, based on the data that's ingested with the run_id.
repeated RunAsset assets = 2 [deprecated=true];
// Start times are automatically calculated based on the data that's ingested with the run_id.
google.protobuf.Timestamp start_time = 3 [deprecated=true];
// Stop times are automatically calculated based on the data that's ingested with the run_id.
optional google.protobuf.Timestamp stop_time = 4 [deprecated=true];
string name = 5;
string description = 6;
repeated string tags = 7;
optional uint32 retention_period_override_in_days = 8;
}
message RunAssetRequest {
string asset_name = 1;
optional string ingestion_config_id = 2;
}
message CreateRunRequest {
// Runs are automatically associated with assets, based on the data that's ingested with the run_id.
repeated RunAssetRequest assets = 1 [deprecated=true];
// Start times are automatically calculated based on the timestamp of the data that's ingested with
// the run_id otherwise it is the time the run was created.
google.protobuf.Timestamp start_time = 2 [deprecated=true];
// Stop times are automatically calculated based on the timestamp of the data that's ingested with
// the run_id otherwise it is the time the run was stopped.
optional google.protobuf.Timestamp stop_time = 3 [deprecated=true];
string name = 4;
string description = 5;
repeated string tags = 6;
optional uint32 retention_period_override_in_days = 7;
// If asset_names are provided, any new data recieved for those assets will automatically
// be associated with this run. This applies even if the run is stopped and new data with
// timestamps within the run start and stop time.
repeated string asset_names = 8;
// organization_id is only required if the user has access to multiple organizations.
string organization_id = 9;
}
message CreateRunResponse {
Run run = 1;
}
message UpdateRunRequest {
string run_id = 1;
// Start times are automatically calculated based on the data that's ingested with the run_id.
optional google.protobuf.Timestamp start_time = 2 [deprecated=true];
// Stop times are automatically calculated based on the data that's ingested with the run_id.
optional google.protobuf.Timestamp stop_time = 3 [deprecated=true];
optional string description = 4;
repeated string tags = 5;
// Runs are automatically associated with assets, based on the data that's ingested with the run_id.
repeated RunAssetRequest assets = 6 [deprecated=true];
optional uint32 retention_period_override_in_days = 7;
}
message UpdateRunResponse {
Run run = 1;
}
message StopRunRequest {
string run_id = 1;
google.protobuf.Timestamp stop_time = 2;
}
message CreateAutomaticRunAssociationForAssetsRequest {
// The ID of the run to associate the asset with.
string run_id = 1 [(google.api.field_behavior) = REQUIRED];
// A list of asset names to automatically associate with the run.
// Any data that is received for these assets will automatically added to the run.
// This applies even if the run has concluded, so long as the new data contains
// timestamps that are between the `start_time` and `stop_time`.
//
// If any of the assets are already associated with a different run whose run
// period (the period between `start_time` and `end_time`) overlaps with the
// requested run period, an error will be returned.
repeated string asset_names = 2 [(google.api.field_behavior) = REQUIRED];
}
message CreateAutomaticRunAssociationForAssetsResponse {
}
message StopRunResponse {
Run run = 1;
}
message IngestProtobufRequest {
string message_type_identifier = 1;
optional string message_type_display_name = 2;
string asset_name = 3;
google.protobuf.Timestamp timestamp = 4;
bytes value = 5;
string run_id = 6;
string namespace = 7;
string organization_id = 8;
// By default, if this request fails to parse for any reason, the request is
// stored in an error queue and the stream continues to accept data. This
// ensures all data is saved, but only valid data is fully ingested. If this
// is set to `true`, any validation errors end the stream and return the error to the client.
bool end_stream_on_validation_error = 9;
}
message SaveProtobufDescriptorRequest {
string message_type_full_name = 1;
string message_type_name = 2;
bytes file_descriptor_set = 3;
string proto_file_name = 4;
string organization_id = 5;
}
Channel Parsing
/**
* These are message and field options that can be used when ingesting protobuf messages. These can be applied to protobuf fields
* and messages before compiling and uploading the descriptor set.
*/
syntax = "proto2";
option go_package = "azimuth/grpc/proto";
package azimuth;
import "google/protobuf/descriptor.proto";
// Enum for different types of tag sources
enum TagSourceType {
DESCENDANT_SOURCES = 0; // Tags can be sourced from descendant fields
SIBLING_SOURCES = 1; // Tags can be sourced from sibling fields
DESCENDANT_AND_SIBLING_SOURCES = 2; // Tags can be sourced from both descendant and sibling fields
}
// Enum for different types of tag targets
enum TagTargetType {
ANCESTOR_TARGETS = 0; // Tags can be applied to ancestor fields
SIBLING_TARGETS = 1; // Tags can be applied to sibling fields
ANCESTOR_AND_SIBLING_TARGETS = 2; // Tags can be applied to both ancestor and sibling fields
}
// Message representing a tag target
message TagTarget {
// Setting this value indicates the allowed sources for tag values.
// If set to SIBLING_SOURCES, tag_sources from descendant proto fields will not be applied.
// If set to DESCENDANT_SOURCES, sibling tag_sources will not be applied.
optional TagSourceType allowed_tag_source = 1;
}
// Message representing a tag source
message TagSource {
// Setting either of these values indicates that the field is a tag source
// and the value can be applied to allowed tag_targets.
// Tag sources apply only to the nearest ancestor (if any) and do not propagate
// to ancestors that are lists or maps or beyond.
// They can apply to both ancestor and sibling fields.
// Indicates which tag_target relationships are allowed to have this tag.
// If set to ANCESTOR_TARGETS, sibling tag_targets will not have this value.
// If set to SIBLING_TARGETS, it will not apply to tag_targets that are ancestors.
optional TagTargetType allowed_tag_target = 1;
// The name of the tag. It defaults to the field name but can be overridden here.
optional string tag_name = 2;
}
// Enum for the source and destination of a map key override
enum MapKeyOverrideType {
MAP_KEY_OVERRIDE_UNSPECIFIED = 0; // Default value, will be ignored
MAP_KEY_OVERRIDE_TARGET = 1; // Map key will be replaced on fields with this type specified
MAP_KEY_OVERRIDE_SOURCE = 2; // Map key will be sourced from fields with this type specified
MAP_KEY_OVERRIDE_REMOVE_KEY = 3; // Map key will be removed from fields with this type specified
MAP_KEY_OVERRIDE_ENUM = 4; // Map key will be replaced with the enum name specified by the display_override_enum value on the field
}
// Enum for the source and destination of an array index override
enum ArrayIndexOverrideType {
ARRAY_INDEX_OVERRIDE_UNSPECIFIED = 0; // Default value, will be ignored
ARRAY_INDEX_OVERRIDE_TARGET = 1; // Array Index will be replaced on fields with this type specified
ARRAY_INDEX_OVERRIDE_SOURCE = 2; // Array Index will be sourced from fields with this type specified
ARRAY_INDEX_OVERRIDE_REMOVE_INDEX = 3; // Array Index will be removed from fields with this type specified
ARRAY_INDEX_OVERRIDE_ENUM = 4; // Array Index will be replaced with the enum name specified by the display_override_enum value on the field
}
// Enum for how to decode bytes fields
enum BytesDecodingType {
BYTES_DECODING_TYPE_UNSPECIFIED = 0; // Default value, will be ignored
BYTES_DECODING_TYPE_UTF8 = 1; // Decode bytes as UTF-8
}
extend google.protobuf.FieldOptions {
// Adding the units FieldOption to a field sets the units displayed on the channel when charted
optional string units = 50001;
// Adding the description FieldOption to a field sets the description for the channel in the chart
optional string description = 50002;
// Adding the tag_target FieldOption indicates that this field can have tag values appended
// to the field name. The tag values will come from descendant or sibling fields in the proto path
// that have the tag_source FieldOption applied.
optional TagTarget tag_target = 50003;
// Adding the tag_source FieldOption allows the value of the annotated field to be added to
// applicable fields that are annotated with tag_target. These fields can be either ancestor fields
// or sibling fields, depending on their respective fields' allowed_tag_source and allowed_tag_target values.
optional TagSource tag_source = 50004;
// Adding the map_key_override_type FieldOption to a map type field with the `MAP_KEY_OVERRIDE_TARGET`
// type indicates that this field can have the map key replaced in the display name by a descendant field.
// Adding this to a primitive type field with the `MAP_KEY_OVERRIDE_SOURCE` indicates that this field
// is the replacement value for an ancestor map's key display. Note that sources within arrays will not apply to
// ancestor maps that contain those arrays. This tag will be ignored if the field is not a map or if there
// are no map_key_override_sources that apply. If multiple apply then the first override will be applied and an
// error will be logged.
optional MapKeyOverrideType map_key_override_type = 50005;
// Adding the bytes_decoding_type FieldOption to a bytes field indicates how the bytes should be decoded
// If not set, the bytes fields will be ignored and an error will be logged.
optional BytesDecodingType bytes_decoding_type = 50006;
// Adding the ignore_field FieldOption with a value of true will cause the field to be ignored when ingesting the proto
optional bool ignore_field = 50007;
// Adding the display_override_enum FieldOption to a field with that has map_key_override_type set to MAP_KEY_OVERRIDE_ENUM
// will look up the enum value in the enum referenced by this field and use that to look up the enum value by number and replace it
// with the enum value name.
optional string display_override_enum = 50008;
// Adding the array_index_override_type FieldOption to a repeated type field with the `ARRAY_INDEX_OVERRIDE_TARGET`
// type indicates that this field can have the array index replaced in the display name by a descendant field.
// Adding this to a primitive type field with the `ARRAY_INDEX_OVERRIDE_SOURCE` type indicates that this field
// is the replacement value for an ancestor array's index display. Note that sources within nested arrays will not apply to
// ancestor arrays that contain those arrays. This tag will be ignored if the field is not an array or if there
// are no `ARRAY_INDEX_OVERRIDE_SOURCE` fields that apply. If multiple apply then the first override will be applied and an
// error will be logged.
optional ArrayIndexOverrideType array_index_override_type = 50009;
}
extend google.protobuf.MessageOptions {
// The message_is_tag_target option indicates that the message will allow its fields to have tags
// added to them. This option only accepts descendant sources.
// It can be helpful to annotate top-level messages with tags from deeper in the proto path.
optional bool message_is_tag_target = 50001;
}