feat(lazer): improve metadata in protobuf (#2856)

* feat(lazer): update metadata proto, rename and clarify docs

* feat(lazer): add DynamicValue parsed type

* test(lazer): add DynamicValue tests

* feat(lazer): add FeedKind to protocol

* feat(lazer): add feed kind to protocol

* fix(lazer): expose dynamic_value protobuf as rust module, move type to protocol

* fix: add FeedKind to AddFeed

* doc(lazer): improve proto comments

Author: Riateche

Cargo.lock

@@ -3,7 +3,7 @@ syntax = "proto3";
 import "google/protobuf/timestamp.proto";
 import "google/protobuf/duration.proto";
 
-package pyth_lazer_transaction;
+package pyth_lazer;
 
 // A dynamically typed value similar to `google.protobuf.Value`
 // but supporting more types.

lazer/publisher_sdk/proto/dynamic_value.proto

@@ -5,11 +5,12 @@ import "google/protobuf/duration.proto";
 import "google/protobuf/empty.proto";
 
 import "dynamic_value.proto";
+import "state.proto";
 
 // If any field documented as `[required]` is not present in the instruction,
 // the instruction will be rejected.
 
-package pyth_lazer_transaction;
+package pyth_lazer;
 
 // Representation of a complete governance instruction. This value will be signed
 // by a governance source.
@@ -130,9 +131,10 @@ message Permissions {
     enum UpdateFeedAction {
         // Required by protobuf. Instruction will be rejected if this value is encountered.
         UPDATE_FEED_ACTION_UNSPECIFIED = 0;
-        UPDATE_FEED_METADATA = 101;
-        ACTIVATE_FEED = 102;
-        DEACTIVATE_FEED = 103;
+        UPDATE_FEED_PROPERTIES = 101;
+        UPDATE_FEED_METADATA = 102;
+        ENABLE_FEED_IN_SHARD = 103;
+        DISABLE_FEED_IN_SHARD = 104;
         REMOVE_FEED = 199;
     }
 
@@ -302,17 +304,33 @@ message SetPublisherActive {
     optional bool is_active = 1;
 }
 
-// Feed is inactive when added, meaning that it will be available to publishers but not to consumers.
+// Add a new feed. Refer to `Feed` message fields documentation.
 message AddFeed {
-    // [required] ID of the feed. Must be unique (within the shard).
+    // [required]
     optional uint32 feed_id = 1;
-    // [required] Feed metadata. Some properties are required (name, exponent, etc.).
-    // Known properties must have the expected type.
-    // Additional arbitrary properties are allowed.
-    // (TODO: document known metadata properties)
-    optional DynamicValue.Map metadata = 2;
-    // IDs of publishers enabled for this feed.
-    repeated uint32 permissioned_publishers = 3;
+    // [required]
+    optional DynamicValue.Map metadata = 3;
+    // [required]
+    optional string name = 101;
+    // [required]
+    optional sint32 exponent = 102;
+    // [required]
+    optional uint32 min_publishers = 103;
+    // [required]
+    optional google.protobuf.Duration min_rate = 104;
+    // [required]
+    optional google.protobuf.Duration expiry_time = 105;
+    // [required]
+    optional string market_schedule = 106;
+    // [required]
+    optional FeedState state = 107;
+    // [required]
+    optional FeedKind kind = 108;
+    // [required]
+    optional bool is_enabled_in_shard = 201;
+
+    // TODO: IDs of publishers enabled for this feed.
+    // repeated uint32 permissioned_publishers = 3;
 }
 
 message UpdateFeed {
@@ -321,43 +339,68 @@ message UpdateFeed {
     // [required]
     // Note: when adding a new variant here, update `Permissions` as well.
     oneof action {
-        UpdateFeedMetadata update_feed_metadata = 101;
-        ActivateFeed activate_feed = 102;
-        DeactivateFeed deactivate_feed = 103;
+        UpdateFeedProperties update_feed_properties = 101;
+        UpdateFeedMetadata update_feed_metadata = 102;
+        EnableFeedInShard enable_feed_in_shard = 103;
+        DisableFeedInShard disable_feed_in_shard = 104;
         google.protobuf.Empty remove_feed = 199;
     }
 }
 
+// Update a feed's properties. The feed will be updated with values present in each field.
+// If a value is not supplied, the corresponding property will remain unchanged.
+// Refer to `Feed` message fields documentation.
+message UpdateFeedProperties {
+    // [optional]
+    optional DynamicValue.Map metadata = 3;
+    // [optional]
+    optional string name = 101;
+    // [optional]
+    optional sint32 exponent = 102;
+    // [optional]
+    optional uint32 min_publishers = 103;
+    // [optional]
+    optional google.protobuf.Duration min_rate = 104;
+    // [optional]
+    optional google.protobuf.Duration expiry_time = 105;
+    // [optional]
+    optional string market_schedule = 106;
+    // [optional]
+    optional FeedState state = 107;
+    // [optional]
+    optional bool is_enabled_in_shard = 201;
+}
+
 message UpdateFeedMetadata {
     // [required] Property name.
     optional string name = 1;
     // [optional] Property value. If unset, the property will be removed.
     optional DynamicValue value = 2;
 }
 
-// Set the feed as active or shedule an activation.
-// If there was already a pending activation or deactivation, it will be cleared
+// Set the feed as enabled in this shard or shedule it for a certain timestamp.
+// If there was already a pending status change, it will be cleared
 // when this governance instruction is processed.
-// Warning: there must never be two feeds with the same name active at the same time
+// Warning: there must never be two feeds with the same name enabled at the same time
 // within a shard group. This cannot be enforced within a shard. When a feed needs to be
-// moved between shards, use `activation_timestamp` and `deactivation_timestamp`
-// to deactivate it in the old shard and activate it in the new shard at the same time.
-message ActivateFeed {
-    // [optional] If provided, the feed will activate at the specified timestamp.
-    // If `activation_timestamp` is already passed or if it's unset,
-    // the feed will be activated immediately when this
+// moved between shards, use `enable_in_shard_timestamp` and `disable_in_shard_timestamp`
+// to disable it in the old shard and enable it in the new shard at the same time.
+message EnableFeedInShard {
+    // [optional] If provided, the feed will be enabled at the specified timestamp.
+    // If `enable_in_shard_timestamp` is already passed or if it's unset,
+    // the feed will be enabled immediately when this
     // governance instruction is processed.
-    optional google.protobuf.Timestamp activation_timestamp = 1;
+    optional google.protobuf.Timestamp enable_in_shard_timestamp = 1;
 }
 
-// Set the feed as inactive or shedule a deactivation.
-// If there was already a pending activation or deactivation, it will be cleared
+// Set the feed as disabled in this shard or shedule it for a certain timestamp.
+// If there was already a pending status change, it will be cleared
 // when this governance instruction is processed.
-// See also: `ActivateFeed` docs.
-message DeactivateFeed {
-    // [optional] If provided, the feed will deactivate at the specified timestamp.
-    // If `deactivation_timestamp` is already passed or if it's unset,
-    // the feed will be deactivated immediately when this
+// See also: `EnableFeedInShard` docs.
+message DisableFeedInShard {
+    // [optional] If provided, the feed will be disabled at the specified timestamp.
+    // If `disable_in_shard_timestamp` is already passed or if it's unset,
+    // the feed will be disabled immediately when this
     // governance instruction is processed.
-    optional google.protobuf.Timestamp deactivation_timestamp = 1;
+    optional google.protobuf.Timestamp disable_in_shard_timestamp = 1;
 }

lazer/publisher_sdk/proto/governance_instruction.proto

@@ -1,5 +1,5 @@
 syntax = "proto3";
-package pyth_lazer_transaction;
+package pyth_lazer;
 
 import "google/protobuf/timestamp.proto";
 
@@ -28,7 +28,7 @@ message FeedUpdate {
     // [required] timestamp when this data was first acquired or generated
     optional google.protobuf.Timestamp source_timestamp = 2;
 
-    // [required] one type of update containing specific data 
+    // [required] one type of update containing specific data
     oneof update {
       PriceUpdate price_update = 3;
       FundingRateUpdate funding_rate_update = 4;

lazer/publisher_sdk/proto/publisher_update.proto

@@ -1,5 +1,5 @@
 syntax = "proto3";
-package pyth_lazer_transaction;
+package pyth_lazer;
 
 import "publisher_update.proto";
 import "governance_instruction.proto";

lazer/publisher_sdk/proto/pyth_lazer_transaction.proto

@@ -1,9 +1,11 @@
 syntax = "proto3";
-package lazer;
+package pyth_lazer;
 
 import "google/protobuf/duration.proto";
 import "google/protobuf/timestamp.proto";
 
+import "dynamic_value.proto";
+
 // All optional fields should always be set unless documented otherwise.
 
 // State of a Pyth Lazer shard.
@@ -45,67 +47,76 @@ message Publisher {
 }
 
 enum FeedState {
-    COMING_SOON = 0; // Default value
+    // Default value. Feeds in this state are not available to consumers.
+    // `COMING_SOON` feeds are expected to become stable in the future.
+    COMING_SOON = 0;
+    // A fully available feed.
     STABLE = 1;
+    // Inactive feeds are not available to consumers.
+    // `INACTIVE` feeds are not expected to become stable again.
     INACTIVE = 2;
 }
 
-// Static data for a feed.
-message FeedMetadata {
-    // [required] ID of the price feed.
-    optional uint32 price_feed_id = 1;
-    // [required] Feed name.
-    optional string name = 2;
-    // [required] Feed symbol.
-    optional string symbol = 3;
-    // [required] Feed description.
-    optional string description = 4;
-    // [required] Feed asset type.
-    optional string asset_type = 5;
+// Feed kind determines the set of data fields available in the feed.
+// It also determines the kind of data accepted from publishers for this feed
+// (`PriceUpdate` or `FundingRateUpdate`).
+enum FeedKind {
+    // Fields: price, best_bid_price, best_ask_price
+    PRICE = 0;
+    // Fields: price, rate.
+    FUNDING_RATE = 1;
+}
+
+// An item of the state describing a feed.
+message Feed {
+    // [required] ID of the feed.
+    optional uint32 feed_id = 1;
+    // Additional state per publisher.
+    // If an eligible publisher is not listed here, the corresponding state should be considered empty.
+    repeated FeedPublisherState per_publisher = 2;
+    // [required] Additional metadata values. These values will be exposed in the APIs, but
+    // are not directly used in the aggregator.
+    optional DynamicValue.Map metadata = 3;
+
+    // [required] A readable feed name. It must be unique across all feeds in the shard.
+    // Used for logs, metrics, feed search API, TradingView API.
+    optional string name = 101;
     // [required] Exponent applied to all price and rate values for this feed.
     // Actual value is `mantissa * 10 ^ exponent`.
     // Restricted to int16.
-    optional sint32 exponent = 6;
-    // [optional] CoinMarketCap ID. Can be absent if there is no CoinMarketCap ID for this symbol.
-    optional uint32 cmc_id = 7;
-    // [optional] Funding rate interval. Only present for funding rate feeds.
-    optional google.protobuf.Duration funding_rate_interval = 8;
+    optional sint32 exponent = 102;
     // [required] Minimal number of publisher prices required to produce an aggregate.
-    optional uint32 min_publishers = 9;
+    optional uint32 min_publishers = 103;
     // [required] Minimal rate of aggregation performed by the aggregator for this feed.
     // Cannot be lower than the shard's top level `State.min_rate`.
-    optional google.protobuf.Duration min_rate = 10;
+    optional google.protobuf.Duration min_rate = 104;
     // [required] Time after which the publisher update is discarded.
-    optional google.protobuf.Duration expiry_time = 11;
-    // [required] If true, the feed is visible to the consumers. This can be used to prepare and verify
-    // new feeds before releasing them. This can also be used to migrate a feed from
-    // one shard to another. If a feed is present in
+    optional google.protobuf.Duration expiry_time = 105;
+    // [required] Market schedule in Pythnet format.
+    optional string market_schedule = 106;
+    // [required] Feed state.
+    optional FeedState state = 107;
+    // [required] Feed kind.
+    optional FeedKind kind = 108;
+
+
+    // [required] Feed status in the current shard. Disabled feeds will not be visible in
+    // the consumer API for the current shard. This setting should only be used
+    // to migrate a feed from one shard to another.
+    //
+    // If a feed is present in
     // multiple shards, it must only be active in one of them at each time.
-    // To enforce this, `pending_activation` and `pending_deactivation` fields
+    // To enforce this, `enable_in_shard_timestamp` and `disable_in_shard_timestamp` fields
     // can be used to deactivate a feed in one shard and activate it in another shard
     // at the same instant.
-    optional bool is_activated = 12;
-    // [optional] ID of the corresponding price feed in Hermes (Pythnet).
-    optional string hermes_id = 13;
-    // [optional] Quote currency of the asset.
-    optional string quote_currency = 14;
-    // [optional] Market schedule in Pythnet format.
-    // If absent, the default schedule is used (market is always open).
-    optional string market_schedule = 15;
-    // [required] Feed state
-    optional FeedState state = 16;
-}
+    optional bool is_enabled_in_shard = 201;
+    // [optional] If present, the aggregator will enable the feed in the current shard
+    // at the specified instant.
+    optional google.protobuf.Timestamp enable_in_shard_timestamp = 202;
+    // [optional] If present, the aggregator will disable the feed in the current shard
+    // at the specified instant.
+    optional google.protobuf.Timestamp disable_in_shard_timestamp = 203;
 
-// An item of the state describing a feed.
-message Feed {
-    optional FeedMetadata metadata = 1;
-    // [optional] If present, the aggregator will activate the feed at the specified instant.
-    optional google.protobuf.Timestamp pending_activation = 2;
-    // [optional] If present, the aggregator will deactivate the feed at the specified instant.
-    optional google.protobuf.Timestamp pending_deactivation = 3;
-    // Additional state per publisher.
-    // If an eligible publisher is not listed here, the corresponding state should be considered empty.
-    repeated FeedPublisherState per_publisher = 4;
     // TODO: list of permissioned publisher IDs.
 }
 

lazer/publisher_sdk/proto/state.proto

@@ -1,5 +1,5 @@
 syntax = "proto3";
-package pyth_lazer_transaction;
+package pyth_lazer;
 
 import "google/protobuf/timestamp.proto";
 import "pyth_lazer_transaction.proto";

lazer/publisher_sdk/proto/transaction_envelope.proto

@@ -10,9 +10,12 @@ repository = "https://github.com/pyth-network/pyth-crosschain"
 pyth-lazer-protocol = { version = "0.9.0", path = "../../sdk/rust/protocol" }
 anyhow = "1.0.98"
 protobuf = "3.7.2"
-serde-value = "0.7.0"
 humantime = "2.2.0"
 tracing = "0.1.41"
+serde = { version = "1.0.219", features = ["derive"] }
+serde_json = "1.0.140"
+derive_more = { version = "2.0.1", features = ["from"] }
+hex = "0.4.3"
 
 [build-dependencies]
 fs-err = "3.1.0"