// Copyright 2021-present PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

// Use ./gen.sh to generate .pb.go files.
syntax = "proto3";

import "./fts.proto";

package enginepb;

message ChangeSet {
  uint64 shardID = 1;
  uint64 shardVer = 2;
  Compaction compaction = 4;
  Flush flush = 5;
  Snapshot snapshot = 6;
  Snapshot initial_flush = 7;
  Split split = 10;
  bool shardDelete = 11;
  // The sequence value is not set by the DB when generating the change set,
  // user have to assign the value. If specified, we check sequence is
  // monotonically increasing.
  uint64 sequence = 12;
  ChangeSet parent = 13;
  IngestFiles ingest_files = 14;
  string property_key = 15;
  bytes property_value = 16;
  bool property_merge = 17;
  TableChange destroy_range = 18;
  TableChange truncate_ts = 19;
  TableChange trim_over_bound = 20;
  Snapshot restore_shard = 21;
  MajorCompaction major_compaction = 22;
  SchemaMeta update_schema_meta = 23;
  ColumnarCompaction columnar_compaction = 24;
  UpdateVectorIndex update_vector_index = 25;
  // Deprecated field. Replaced by clear_columnar_with_restore_version.
  bool clear_columnar = 26;
  // Used for shard meta persist by diff.
  Snapshot snapshot_diff = 27;
  ClearColumnar clear_columnar_with_restore_version = 28;
  FtsUpdate fts_update = 29;
}

// A set of small changesets of the same peer.
message ChangeSets { repeated ChangeSet change_sets = 1; }

message Compaction {
  int32 cf = 1;
  uint32 level = 2;
  repeated TableCreate tableCreates = 3;
  repeated uint64 topDeletes = 4;
  repeated uint64 bottomDeletes = 5;
  bool conflicted = 6;
  repeated BlobCreate blobTables = 7;
}

message MajorCompaction {
  TableChange sstableChange = 1;
  repeated BlobCreate newBlobTables = 2;
  repeated uint64 oldBlobTables = 3;
  bool conflicted = 4;
  bool update_inner_key_offset = 5;
}

message SchemaMeta {
  uint32 keyspace_id = 1;
  uint64 file_id = 2;
  int64 version = 3;
  uint64 restore_version = 4;
}

message ColumnarCompaction {
  TableChange columnar_change = 1;
  uint64 snap_version = 2;
  repeated uint64 row_l0s = 3;
  uint32 target_level = 4;
  repeated int64 columnar_table_ids = 5;
  repeated int64 columnar_table_ids_to_clear = 6;
  bool is_manual_major_compaction = 7;
}

message UpdateVectorIndex {
  int64 table_id = 1;
  int64 index_id = 2;
  int64 col_id = 3;
  repeated VectorIndexFile added = 4;
  repeated uint64 removed = 5;
  uint64 snap_version = 6;
}

message Flush {
  L0Create l0Create = 1;
  Properties properties = 2;
  uint64 version = 3;
  uint64 max_ts = 5;
  // used for txn file transaction which may be too large to flush a single L0
  // file.
  repeated L0Create l0Creates = 6;
}

message Snapshot {
  reserved 14; // deprecated columnar_snap_version.
  bytes outer_start = 1;
  bytes outer_end = 2;
  Properties properties = 3;
  repeated L0Create l0Creates = 5;
  repeated TableCreate tableCreates = 6;
  uint64 baseVersion = 7;
  uint64 data_sequence = 8;
  repeated BlobCreate BlobCreates = 9;
  uint64 max_ts = 10;
  uint32 inner_key_off = 11;
  repeated ColumnarCreate columnarCreates = 12;
  SchemaMeta schema_meta = 13;
  repeated uint64 unconverted_l0s = 15;
  repeated VectorIndex vector_indexes = 16;
  repeated int64 columnar_table_ids = 17;
  uint64 columnar_l2_snap_version = 18;

  // Below are FTS related fields
  repeated TableIndexId fts_indexes = 40;
  uint64 fts_l0_snap_version = 41;
  // Columnar L0 ids that still need incremental FTS processing after merge.
  repeated uint64 fts_pending_l0_ids = 42;
  repeated FtsTrackedIndexMeta fts_index_metas = 43;
  repeated FtsPackedFileInfo fts_l0_files = 45;
  repeated FtsPackedFileInfo fts_l1_files = 46;
  repeated FtsDedFileInfo fts_l2_files = 47;
}

message L0Create {
  uint64 ID = 1;
  bytes smallest = 2;
  bytes biggest = 3;
  uint32 size = 4;
}

message BlobCreate {
  uint64 ID = 1;
  bytes smallest = 2;
  bytes biggest = 3;
  uint32 meta_offset = 4;
}

message TableCreate {
  reserved 6; // deprecated field.
  uint64 ID = 1;
  uint32 level = 2;
  int32 CF = 3;
  bytes smallest = 4;
  bytes biggest = 5;
  uint32 meta_offset = 7;
}

message TableDelete {
  uint64 ID = 1;
  uint32 level = 2;
  int32 CF = 3;
}

message ColumnarCreate {
  uint64 ID = 1;
  uint32 level = 2;
  bytes smallest = 3;
  bytes biggest = 4;
  uint32 meta_offset = 5;
  uint64 snap_version = 6;
}

message ColumnarDelete {
  uint64 ID = 1;
  uint32 level = 2;
}

message Split {
  repeated Properties newShards = 1;
  repeated bytes Keys = 3;
}

message IngestFiles {
  repeated L0Create l0Creates = 1;
  repeated TableCreate tableCreates = 2;
  Properties properties = 3;
  repeated BlobCreate BlobCreates = 4;
  uint64 max_ts = 5;
}

message Properties {
  uint64 shardID = 1;
  repeated string keys = 2;
  repeated bytes values = 3;
}

message TableChange {
  // Tables that deleted by destroy range / truncate ts.
  repeated TableDelete tableDeletes = 1;
  // Tables that created by destroy range / truncate ts.
  repeated TableCreate tableCreates = 2;
  // For in-place compaction that each deleted file id followed by a created
  // file id.
  repeated uint64 file_ids_map = 3;
  // Columnar Tables that deleted by destroy range / truncate ts / compaction /
  // trim over bound.
  repeated ColumnarDelete columnarDeletes = 4;
  // Columnar Tables that created by destroy range / truncate ts / compaction /
  // trim over bound.
  repeated ColumnarCreate columnarCreates = 5;
}

message TxnFileRefs { repeated TxnFileRef txn_file_refs = 1; }

message TxnFileRef {
  uint64 start_ts = 1;
  repeated uint64 chunk_ids = 2;
  // The value of version is data_sequence for lock CF, commit_ts for write CF.
  uint64 version = 3;
  // empty for Lock CF,
  bytes user_meta = 4;
  // encoded txn_types::Lock without short value.
  bytes lock_val_prefix = 5;
  // Not used for now, but maybe useful for merge/split
  uint64 shard_ver = 6;

  // Lower bound without keyspace prefix.
  bytes inner_lower_bound = 7;
  // Upper bound (exclusive) without keyspace prefix.
  bytes inner_upper_bound = 8;
}

message Schema {
  int64 table_id = 1;
  // serialized tipb::ColumnInfo.
  repeated bytes columns = 2;
  repeated int64 pk_col_ids = 3;
  repeated VectorIndexDef vector_indexes = 4;
  repeated FullTextIndexDef fulltext_indexes = 10;
  repeated string keys = 5;
  repeated bytes values = 6;
  repeated Partition partitions = 7;
  int64 max_col_id = 8;
}

message Partition {
  int64 id = 1;
  repeated string keys = 2;
  repeated bytes values = 3;
}

message VectorIndex {
  int64 table_id = 1;
  int64 index_id = 2;
  int64 col_id = 3;
  repeated VectorIndexFile files = 4;
  uint64 snap_version = 5;
}

message VectorIndexFile {
  uint64 id = 1;
  uint64 snap_version = 2;

  // smallest handle of the file.
  bytes smallest = 3;
  // biggest handle of the file.
  bytes biggest = 4;
  uint32 meta_offset = 6;

  // Deprecated field. (old meta_offset)
  reserved 5;
}

message VectorIndexDef {
  int64 index_id = 1;
  int64 col_id = 2;
  string index_kind = 3;
  repeated string spec_keys = 4;
  repeated bytes spec_values = 5;
}

message FtsPackedFileInfo {
  uint64 id = 1;
  uint32 meta_offset = 2;

  bytes smallest = 3;
  bytes biggest = 4;

  // Only available for FTS L0 files.
  uint64 snap_version = 5;
}

message FtsDedFileInfo {
  uint64 id = 1;
  uint32 meta_offset = 2;

  // Row key range covered by this file.
  bytes smallest = 3;
  bytes biggest = 4;
}

message FtsUpdate {
  // Optional watermark update
  uint64 set_snap_version = 1;
  // Addition - add new indexes to existing configuration
  repeated TableIndexId add_tracked_indexes = 2;
  // Removal - remove tracked indexes
  repeated TableIndexId remove_tracked_indexes = 11;
  // Addition - add new FTS L0 files
  repeated FtsPackedFileInfo l0_add_files = 3;
  // Removal - remove old FTS L0 files
  repeated uint64 l0_remove_files = 4;
  // Addition - add new FTS L1 files
  repeated FtsPackedFileInfo l1_add_files = 5;
  // Removal - remove old FTS L1 files
  repeated uint64 l1_remove_files = 6;
  // Addition - add new FTS L2 dedicated files
  repeated FtsDedFileInfo l2_add_files = 7;
  // Removal - remove old FTS L2 dedicated files
  repeated uint64 l2_remove_files = 8;
  // Clear all tracked FTS indexes and files
  bool clear_all = 9;
  // Removal - remove pending columnar l0 files which are now indexed
  // (this field allows ids even not in pending list, they will be skipped)
  repeated uint64 pending_columnar_l0_remove_files = 10;
  // Addition - add tracked index metadata with explicit physical layout.
  repeated FtsTrackedIndexMeta add_tracked_index_metas = 12;
}

message ClearColumnar { uint64 restore_version = 1; }

message MetaPackHeader { uint32 version = 1; }
message PackedBackup {
  repeated ChangeSet shards = 1;
  uint64 cluster_id = 2;
  uint64 backup_ts = 3;
  uint64 safe_ts = 4;
  uint32 keyspace_id = 5;
  uint64 resolved_ts = 8;
  uint64 engine_size = 9;

  string content_bucket = 7;

  string keyspace_name = 11;
  uint64 keyspace_size = 10;

  // For not-yet-unpacked backups only.
  repeated FileRef content_file_refs = 6;

  // For unpacked backups only.
  bool unpacked = 12;
  string unpacked_from = 13;

  // Raw value of the PD key
  // /pd/{cluster_id}/keyspaces/meta/{keyspace_id:08}, containing a serialized
  // keyspacepb.KeyspaceMeta. Empty when the source backup lacks this key.
  bytes keyspace_meta = 14;
}

message FileRef {
  uint32 file_type = 1;
  uint64 file_id = 2;
  string file_abs_path = 3;
}

// Top-level metadata that describes the encryption info for a keyspace.
message EncryptionMeta {
  uint32 keyspace_id = 1;
  // The current encryption epoch, containing the active data key.
  EncryptionEpoch current = 2;
  // The master key used to encrypt the data keys in this meta file.
  MasterKey master_key = 3;
  // All known data keys for the keyspace, indexed by their data_key_id.
  map<uint32, DataKey> data_keys = 4;
  // Previous encryption epochs (ordered by creation time), useful for auditing
  // or key recovery.
  repeated EncryptionEpoch history = 5;
}

// Describes one encryption epoch: a versioned period during which a specific data key
// is used to encrypt user files.
message EncryptionEpoch {
  // The file ID of the encryption meta associated with this epoch.
  uint64 file_id = 1;
  // The data key ID used to encrypt the file.
  uint32 data_key_id = 2;
  // The timestamp when this epoch was created (Unix time, seconds).
  uint64 created_at = 3;
}

// Describes the encrypted master key and its associated CMEK metadata.
// With the CMEK info here it's possible to decrypt the master key.
message MasterKey {
  // The KMS vendor (e.g., "aws-kms").
  string vendor = 1;
  // CMEK identifier.
  string cmek_id = 2;
  // The region of the KMS service, if applicable.
  string region = 3;
  // The endpoint of the KMS service, if applicable.
  string endpoint = 4;
  // The encrypted master key.
  bytes ciphertext = 5;
  // The KMS encryption context, if applicable.
  map<string, string> encryption_context = 6;
}

// Describes a data key that is used to encrypt user data. The data key itself
// is encrypted by the master key.
message DataKey {
  // The encrypted data key.
  bytes ciphertext = 1;
}