twap/ annualized vol and documentation update

Author: richardbrooks2000

dashboard/dashboard.js

@@ -40,6 +40,8 @@ class Prices
     let expo = -pxa.price_exponent;
     let px   = res.price * this.fact[expo];
     let conf = res.conf * this.fact[expo];
+    let twap = res.twap * this.fact[expo];
+    let avol = res.avol * this.fact[expo];
     let color = 'cornsilk';
     if ( res.status == 'unknown' ) {
       color = '#c0392b';
@@ -50,6 +52,8 @@ class Prices
     row.cells[col++].style.color = color;
     this.draw_cell( row.cells[col++], px.toFixed(expo), color );
     this.draw_cell( row.cells[col++], conf.toFixed(expo), color );
+    this.draw_cell( row.cells[col++], twap.toFixed(expo), color );
+    this.draw_cell( row.cells[col++], avol.toFixed(expo), color );
     this.draw_cell( row.cells[col++], res.status, color );
     this.draw_cell( row.cells[col++], res.valid_slot, color );
     this.draw_cell( row.cells[col++], res.pub_slot, color );
@@ -98,6 +102,8 @@ class Prices
         row.appendChild( document.createElement( 'TD' ) );
         row.appendChild( document.createElement( 'TD' ) );
         row.appendChild( document.createElement( 'TD' ) );
+        row.appendChild( document.createElement( 'TD' ) );
+        row.appendChild( document.createElement( 'TD' ) );
         row.appendChild( this.get_title( att, 'tenor') );
         row.appendChild( this.get_title( att, 'quote_currency') );
         row.appendChild( this.get_title( att, 'description') );

dashboard/index.html

@@ -17,6 +17,8 @@ <h2>pyth dashboard
           <th>ptype</th>
           <th width=100px>price</th>
           <th width=100px>conf</th>
+          <th width=100px>twap</th>
+          <th width=100px>avol</th>
           <th width=80>status</th>
           <th>valid_slot</th>
           <th>pub_slot</th>

doc/getting_started.md

@@ -1,6 +1,6 @@
 # Getting started with publishing via pyth-client
 
-The pyth-client repo consists of a C++ library (libpc.a), a command-line administration tool (pyth) and a json/websocket-based server (pythd).
+The pyth-client repo consists of a C++ library (libpc.a), a command-line administration tool (pyth), a json/websocket-based server (pythd) and a gateway/proxy server for price publishing (pyth_tx).
 
 You can integrate with libpc directly in your application. See `pctest/test_publish.cpp` for an example. Or, you can integrate with the pythd server via json/websockets. See `pctest/test_publish.py` for an example.
 
@@ -29,7 +29,7 @@ This will output the public key in base58 encoding and will look something like:
 5rYvdyWAunZgD2EC1aKo7hQbutUUnkt7bBFM6xNq2z7Z
 ```
 
-Once permissioned, you need two additional public keys in the key-store. The id of the mapping-account that contains the directory listing of the on-chain symbols and the id of the on-chain oracle program that you use to publish prices.  Mapping and program accounts are maintained in three separate environments: pythnet, devnet and forthcoming mainnet-beta.
+Once permissioned, you need three additional public keys in the key-store. The id of the mapping-account that contains the directory listing of the on-chain symbols, the id of the on-chain oracle program that you use to publish prices and the id of the parameter-account that contains various math look-up tables used to compute the aggregate price.  Mapping, program and parameter accounts are maintained in three separate environments: pythnet, devnet and forthcoming mainnet-beta.
 
 Use the init_key_store.sh script to initialize these account keys:
 
@@ -41,17 +41,19 @@ KENV=pythnet  # or devnet or mainnet-beta
 
 ```
 
-This creates two files: `$KDIR/program_key.json` and `$KDIR/mapping_key.json`.
+This creates three files: `$KDIR/mapping_key.json`, `$KDIR/program_key.json`,  and `$KDIR/param_key.json`.
 
 Once permissioned, you can test your setup by running the test_publish.cpp example program for publishing and subscribing to two test symbols.  To test publishing on pythnet you need to run:
 
 
 ```
-KHOST=44.232.27.44 # or api.devnet.solana.com
+KHOST=44.232.27.44
 ./test_publish -k $KDIR -r $KHOST -t $KHOST
 ```
 
-The certification environment (pythnet) can be found at the IP address 44.232.27.44. Please also provide to the administrator the IP address you plan to publish to pythnet from so that it can be added to the pythnet firewall.  Solana devnet can be found at: devnet.solana.com. No additional permissioning is required to publish to devnet.
+The -r and -t options correspond to the locations of required solana validator and pyth_tx server instances.
+
+The certification environment (pythnet) can be found at the IP address 44.232.27.44 where the test validator and pyth_tx instances run. Please also provide to the administrator the IP address you plan to publish to pythnet from so that it can be added to the pythnet firewall.
 
 You can also publish to solana using the pythd server. Start up the server using the same key-store directory and host specification:
 
@@ -66,6 +68,25 @@ Run the test_publish.py example program on the same host to connect to the pythd
 
 ```
 
+## Running the pyth_tx server
+
+The pyth-client API connects to two separate services via TCP. These include the solana validator and pyth_tx servers. The pyth_tx server is included in the pyth-client repository.
+
+The solana rpc interface is used for tracking slot and account updates. The pyth_tx server is used to submit price transactions directly to solana leader nodes via UDP.
+
+pyth_tx runs as a separate server rather than being integrated directly into the client library to avoid a publisher from having to submit UDP datagrams to hundreds of different IP address from within an organization's firewall. Instead, a pyth_tx server can be deployed outside the firewall and a publisher clienti, running inside the firewall, can make a single TCP connection to it on a dedicated port.
+
+You can run your own pyth_tx instance (recommended) or connect to provided server instances running against pythnet and mainnet-beta environments.
+
+Start up the pyth_tx server as follows:
+
+```
+./pyth_tx -r $KHOST
+```
+
+The -r option points to the IP address of a solana validator node or rpc end-point. It can be the same or different IP address to that passed to test_publish or pythd as long as it corresponds to the same environment.
+
+
 ## Running the dashboard
 
 The pythd server also exports a dashboard web page for watching the ticking pyth prices.  The public key you create above does not need publish-permission to watch the ticking pyth prices.  To activate the dashboard page include an additional `-w` argument to pythd pointing at the html/javscript code directory:

doc/migrating_to_version_2.md

@@ -0,0 +1,45 @@
+# Migrating to pyth version 2
+
+Version 2 of the pyth on-chain program contains the full implementation of the aggregate price and confidence interval calculation. There have been changes to the price account data structure to make room for additional outputs including twap and volatility. A new account, known as the parameter account, has been introduced and contains math lookup tables that are required to compute the aggregate price.
+
+A new set of mapping, product, price and program accounts has been deployed to devnet for version 2 and all publishers and subscribers should to migrate to these new accounts.
+
+## Migrating publishers
+
+Publishers should upgrade to the latest version of the pyth-client API (version 2).  Two git branches, corresponding to versions 1 and 2, labelled v1 and v2, have been established to facilitate this.
+
+Publishers need to migrate to new account keys.  You should preserve your existing publishing key but update the mapping_key.json, program_key.json and param_key.json files in your key store directory. The simplest way to do this is to create a new directory, copy your publish_key_pair.json file to that directory and then run init_key_store.sh to generate the new account keys:
+
+```
+KENV=pythnet  # or devnet or mainnet-beta
+# KDIR= old key-store directory
+# NDIR= new key-store directory
+
+# copy publish key over to new directory
+mkdir $NDIR
+cp $KDIR/publish_key_pair.json $NDIR
+
+# initialize new directory
+../pctest/init_key_store.sh $KENV $NDIR
+```
+
+The client API is also versioned and will fail to run against on-chain accounts that are set to a different version, so version 1 accounts must run against the v1 branch of the software and version 2 accounts must run against the v2 branch of the software.
+
+## Migrating subscribers
+
+On-chain subscribers should migrate to the new accounts defined above in the publishers section.  New price accounts contain additional values derived from the aggregate price including twap and volatility as well as space for additional contributing publishers, now upgraded to 32 publishers from 16 in version 1:
+
+```
+// from program/src/oracle/oracle.h:
+typedef struct pc_price
+{
+//...
+  int64_t         twap_;              // time-weighted average price
+  int64_t         avol_;              // annualized price volatility
+//....
+  pc_price_comp_t comp_[PC_COMP_SIZE];// component prices
+} pc_price_t;
+
+```
+
+Please note that the version field in all version 2 account headers now read 2 instead of 1.

pc/request.cpp

@@ -1995,6 +1995,8 @@ price::price( const pub_key& acc, product *prod )
   pub_idx_( (unsigned)-1 ),
   apx_( 0 ),
   aconf_( 0 ),
+  twap_( 0 ),
+  avol_( 0 ),
   valid_slot_( 0 ),
   pub_slot_( 0 ),
   lamports_( 0UL ),
@@ -2004,7 +2006,6 @@ price::price( const pub_key& acc, product *prod )
   sched_( this )
 {
   __builtin_memset( &cpub_, 0, sizeof( cpub_ ) );
-  __builtin_memset( &dpx_, 0, sizeof( dpx_ ) );
   areq_->set_account( &apub_ );
   sreq_->set_account( &apub_ );
   preq_->set_account( &apub_ );
@@ -2072,10 +2073,14 @@ int64_t  price::get_price() const
   return apx_;
 }
 
-int64_t price::get_deriv( deriv_type dtype ) const
+int64_t price::get_twap() const
 {
-  unsigned i = (unsigned)dtype - 1;
-  return i<PC_DERIV_SIZE?dpx_[i] : 0L;
+  return twap_;
+}
+
+uint64_t price::get_ann_volatility() const
+{
+  return avol_;
 }
 
 uint64_t price::get_conf() const
@@ -2227,13 +2232,12 @@ void price::init_price( pc_price_t *pupd )
   version_ = pupd->ver_;
   apx_     = pupd->agg_.price_;
   aconf_   = pupd->agg_.conf_;
+  twap_    = pupd->twap_;
+  avol_    = pupd->avol_;
   sym_st_  = (symbol_status)pupd->agg_.status_;
   pub_slot_   = pupd->agg_.pub_slot_;
   valid_slot_ = pupd->valid_slot_;
   cnum_ = pupd->num_;
-  for( unsigned i=0; i < (unsigned)deriv_type::e_last_deriv_type-1;++i ) {
-    dpx_[i] = pupd->drv_[i];
-  }
   for( unsigned i=0; i != cnum_; ++i ) {
     if ( !pc_pub_key_equal( &cpub_[i], &pupd->comp_[i].pub_ ) ) {
       pc_pub_key_assign( &cpub_[i], &pupd->comp_[i].pub_ );
@@ -2337,12 +2341,11 @@ void price::update( T *res )
   if ( valid_slot_ != pupd->valid_slot_ || valid_slot_ == 0UL ) {
     apx_  = pupd->agg_.price_;
     aconf_ = pupd->agg_.conf_;
+    twap_  = pupd->twap_;
+    avol_   = pupd->avol_;
     sym_st_ = (symbol_status)pupd->agg_.status_;
     pub_slot_ = pupd->agg_.pub_slot_;
     valid_slot_ = pupd->valid_slot_;
-    for( unsigned i=0; i < (unsigned)deriv_type::e_last_deriv_type-1;++i ) {
-      dpx_[i] = pupd->drv_[i];
-    }
 
     // capture aggregate price and components to disk
     mgr->write( (pc_pub_key_t*)apub_.data(), (pc_acc_t*)pupd );
@@ -2444,3 +2447,4 @@ void price_sched::schedule()
 {
   on_response_sub( this );
 }
+

pc/request.hpp

@@ -694,7 +694,8 @@ namespace pc
     uint64_t      get_conf() const;
     symbol_status get_status() const;
     uint64_t      get_lamports() const;
-    int64_t       get_deriv( deriv_type ) const;
+    int64_t       get_twap() const;
+    uint64_t      get_ann_volatility() const;
 
     // get publishers
     unsigned get_num_publisher() const;
@@ -751,10 +752,11 @@ namespace pc
     uint32_t               pub_idx_;
     int64_t                apx_;
     uint64_t               aconf_;
+    int64_t                twap_;
+    uint64_t               avol_;
     uint64_t               valid_slot_;
     uint64_t               pub_slot_;
     uint64_t               lamports_;
-    int64_t                dpx_[PC_DERIV_SIZE];
     int32_t                aexpo_;
     uint32_t               cnum_;
     product               *prod_;

pc/rpc_client.hpp

@@ -34,19 +34,6 @@ namespace pc
   price_type str_to_price_type( str );
   str price_type_to_str( price_type );
 
-  // type of derived (from price) calculation
-  enum class deriv_type
-  {
-    e_unknown    = PC_DTYPE_UNKNOWN,
-    e_twap       = PC_DTYPE_TWAP,
-    e_volatility = PC_DTYPE_VOLATILITY,
-
-    e_last_deriv_type
-  };
-
-  deriv_type str_to_deriv_type( str );
-  str deriv_type_to_str( deriv_type );
-
   // current symbol trading status
   enum class symbol_status
   {

pc/user.cpp

@@ -353,6 +353,8 @@ void user::on_response( price *rptr, uint64_t idx )
   jw_.add_key( "result", json_wtr::e_obj );
   jw_.add_key( "price", rptr->get_price() );
   jw_.add_key( "conf", rptr->get_conf() );
+  jw_.add_key( "twap", rptr->get_twap() );
+  jw_.add_key( "avol", rptr->get_ann_volatility() );
   jw_.add_key( "status", symbol_status_to_str( rptr->get_status() ) );
   jw_.add_key( "valid_slot", rptr->get_valid_slot() );
   jw_.add_key( "pub_slot", rptr->get_pub_slot() );

pcapps/pyth_csv.cpp

@@ -72,6 +72,8 @@ void csv_print::print_header()
             << ",status"
             << ",price"
             << ",conf"
+            << ",twap"
+            << ",avol"
             << ",valid_slot"
             << ",pub_slot";
   for(unsigned i=0; i != PC_COMP_SIZE; ++i ) {
@@ -130,6 +132,8 @@ void csv_print::parse_price( replay& rep )
   std::cout.write( sstr.str_, sstr.len_ );
   std::cout << ',' << ptr->agg_.price_
             << ',' << ptr->agg_.conf_
+            << ',' << ptr->twap_
+            << ',' << ptr->avol_
             << ',' << ptr->valid_slot_
             << ',' << ptr->agg_.pub_slot_;
   for( unsigned i=0; i != ptr->num_; ++i ) {

pctest/init_key_store.sh

@@ -45,3 +45,4 @@ echo $MAP_KEY > $KDIR/mapping_key.json
 check "chmod 0400 $KDIR/mapping_key.json"
 echo $PRM_KEY > $KDIR/param_key.json
 check "chmod 0400 $KDIR/param_key.json"
+check "chmod 0700 $KDIR"