Patch: Working with PVOutput API Batch size and Rate limits

[poolroom]

Hi,

Just wanted to share a patch I've put together to get smatool to work with the limits for uploading to PVOutput.

For non-donors PVOutput limits uploads via their API to batch sizes of 30 data points per API call and a maximum of 60 data points per hour. For donors these numbers go up to 100 and 300 respectively.

My patch simply keeps a count of the uploads made, will limit batch sizes appropriately and will sleep smatool for an hour when the hourly limit is reached. This means you can run smatool over a large time period and it will methodically plug away and get all the data uploaded. Of course when you run smatool like this it'll take some hours to get everything uploaded, but it does get there.

I've also put a basic logic check on the size of the API request being sent to PVOutput. This fixes a potential seg fault with smatool when trying to upload large batch sizes.

I recommend for anyone starting new with PVOutput and wanting to upload a back log of data to donate a few dollars to them. This allows you to take advantage of the increased upload limits.

I've tested this with a run of 2 months of data which is as much as my inverter seems to keep. It ran perfectly.

Once patched smatool will read the Rate and Batch numbers via the config file, or accept them via the command line. If nothing is provided it will work with the current default limits of 30 and 60. So the patch can be added to smatool without requiring further updates.

If you want to keep things need you can update the config file, my example of the additional lines to add to the file are:

# Default rate limit of 60 API requests per hour. Or 300 if a donator.

PVRateLimit     60

# Default batch size of 30 API data points per request. Or 100 if a donator.

PVBatchSize     30

The smatool.c diff is as attached and following.

I haven't checked if this functionality exists in other patches or updates elsewhere.

My thanks to all for their work on the tool.

smatool.c diff

--- smatool.c   2013-09-03 11:39:47.358246987 +1000

+++ smatool.c.org       2013-09-01 17:15:53.220915935 +1000

@@ -63,8 +63,6 @@

     char PVOutputURL[80];           /*--pvouturl     -url   */

     char PVOutputKey[80];           /*--pvoutkey     -key   */

     char PVOutputSid[20];           /*--pvoutsid     -sid   */

-    int PVRateLimit;                /*--pvoutrate    -rate  */

-    int PVBatchSize;                /*--pvoutbatch   -batch */

     char Setting[80];               /* inverter model data  */

     unsigned char InverterCode[4];  /* Unknown code inverter specific*/

     unsigned int ArchiveCode;       /* Code for archive data */

@@ -701,9 +699,7 @@

     //Check if all PVOutput variables are set

     if(( strlen(conf->PVOutputURL) > 0 )

      &&( strlen(conf->PVOutputKey) > 0 )

-     &&( strlen(conf->PVOutputSid) > 0 )

-     &&( conf->PVRateLimit > 0 )

-     &&( conf->PVBatchSize > 0 ))

+     &&( strlen(conf->PVOutputSid) > 0 ))

         (*post)=1;

     else

         (*post)=0;

@@ -778,8 +774,6 @@

     strcpy( conf->PVOutputURL, "http://pvoutput.org/service/r2/addstatus.jsp" );

     strcpy( conf->PVOutputKey, "" );

     strcpy( conf->PVOutputSid, "" );

-    conf->PVRateLimit=60;

-    conf->PVBatchSize=30;

     conf->InverterCode[0]=0;

     conf->InverterCode[1]=0;

     conf->InverterCode[2]=0;

@@ -851,10 +845,6 @@

                        strcpy( conf->PVOutputKey, value );

                     if( strcmp( variable, "PVOutputSid" ) == 0 )

                        strcpy( conf->PVOutputSid, value );

-                    if( strcmp( variable, "PVRateLimit" ) == 0 )

-                       conf->PVRateLimit =  atoi(value);

-                    if( strcmp( variable, "PVBatchSize" ) == 0 )

-                       conf->PVBatchSize =  atoi(value);

                 }

             }

         }

@@ -993,11 +983,9 @@

     printf( "       --INSTALL                           install mysql data tables\n");

     printf( "       --UPDATE                            update mysql data tables\n");

     printf( "PVOutput.org (A free solar information system) Configs\n" );

-    printf( "  -url,    --pvouturl PVOUTURL             pvoutput.org live url\n");

-    printf( "  -key,    --pvoutkey PVOUTKEY             pvoutput.org key\n");

-    printf( "  -sid,    --pvoutsid PVOUTSID             pvoutput.org sid\n");

-    printf( "  -rate,   --pvoutrate PVOUTRATE           pvoutput.org rate limit\n");

-    printf( "  -batch,  --pvoutbatch PVOUTBATCH         pvoutput.org batch limit\n");

+    printf( "  -url,  --pvouturl PVOUTURL               pvoutput.org live url\n");

+    printf( "  -key,  --pvoutkey PVOUTKEY               pvoutput.org key\n");

+    printf( "  -sid,  --pvoutsid PVOUTSID               pvoutput.org sid\n");

     printf( "  -repost                                  verify and repost data if different\n");

     printf( "\n\n" );

 }

@@ -1126,18 +1114,6 @@

                 strcpy(conf->PVOutputSid,argv[i]);

             }

         }

-        else if ((strcmp(argv[i],"-rate")==0)||(strcmp(argv[i],"--pvoutrate")==0)) {

-            i++;

-            if(i<argc){

-                conf->PVRateLimit = atoi(argv[i]);

-            }

-        }

-        else if ((strcmp(argv[i],"-batch")==0)||(strcmp(argv[i],"--pvoutbatch")==0)) {

-            i++;

-            if(i<argc){

-                conf->PVBatchSize = atoi(argv[i]);

-            }

-        }

         else if ((strcmp(argv[i],"-h")==0) || (strcmp(argv[i],"--help") == 0 )) {

             PrintHelp();

             return( -1 );

@@ -1208,7 +1184,7 @@

   return -1;

 }

-void post_interval_data(char *pvOutputUrl, char *pvOutputKey, char *pvOutputSid, int PVRateLimit, int PVBatchSize, int repost, char *datefrom, char *dateto, loglevel_t loglevel)

+void post_interval_data(char *pvOutputUrl, char *pvOutputKey, char *pvOutputSid, int repost, char *datefrom, char *dateto, loglevel_t loglevel)

 {

   time_t prior = time(NULL) - ( 60 * 60 * 24 * 14 ); //up to 14 days before now (r2 service)

   struct tm from_datetime = *(localtime( &prior ) );

@@ -1240,12 +1216,9 @@

   struct tm start_datetime, this_datetime;

   int more_rows = 1;

   int string_end = 0;

-  int urlmaxlen = 2048;

-  char posturl[urlmaxlen];

+  char posturl[2048];

   long startOfDayWh = 0;

   int curlResult = -1;

-  int curlHourCount = 0;

-  int curlPostTotal = 0;

   while( more_rows )

   {

     if( 0 == rows_processed )

@@ -1264,37 +1237,22 @@

     rows_processed++;

     more_rows = db_row_next( row ); //db_next_row returns 0 if we cannot move to next row in result set, 1 otherwise

-    //r2 service - can process a maximum rows at a time

-    if( ( PVBatchSize <= rows_processed ) ||

-        ( PVRateLimit <= ( curlHourCount + rows_processed )) ||

-        ( 0 == more_rows ) ||

-        ( 50 >= ( urlmaxlen - strlen( posturl ))) )

+    //r2 service - can process upto 30 rows at a time

+    if( 30 == rows_processed || 0 == more_rows  )

     {

       //if post requires last ; to be stripped... posturl[string_end] = '\0';

       curlResult = curl_post_this_query(posturl, pvOutputKey, pvOutputSid, loglevel);

-      curlHourCount = curlHourCount + rows_processed;

-      curlPostTotal = curlPostTotal + rows_processed;

-

       if ( curlResult == 0 )

       {

-        db_set_data_posted(&start_datetime, &this_datetime );  //date range covering possibly 1, but at most PVBatchSize values

+        db_set_data_posted(&start_datetime, &this_datetime );  //date range covering possibly 1, but at most 30, values

         rows_processed = 0;

         sleep(2); //pvoutput api says we can't post more than once a second.

       } else {

           log_error( "CURL post failed, CURL result was %d",curlResult );

       }

-

-      if (( PVRateLimit <= curlHourCount ) && ( 1 == more_rows ))

-      {

-        log_info( "%d API requests made = PVOutput limit. Sleeping for 1 hour..." , curlHourCount);

-        curlHourCount = 0;

-        sleep(3600); // Sleep for 60 mins

-      }

     }

   }

-  log_info( "Total %d data points uploaded." , curlPostTotal);

-

   db_row_handle_free( row );

 }

@@ -2175,7 +2133,7 @@

     archdatalen=0;

     free(last_sent);

     if ((post ==1)&&(mysql==1)&&(error==0)){

-      post_interval_data( conf.PVOutputURL, conf.PVOutputKey, conf.PVOutputSid, conf.PVRateLimit, conf.PVBatchSize, repost, datefrom, dateto, loglevel);

+      post_interval_data( conf.PVOutputURL, conf.PVOutputKey, conf.PVOutputSid, repost, datefrom, dateto, loglevel);

     }

 }

[poolsolar]

Sorry, I had the order of that diff the wrong way around.

The corrected diff is attached and below.

--- smatool.c.org 2013-09-03 17:45:06.424555107 +1000

+++ smatool.c 2013-09-03 17:45:33.853556845 +1000

@@ -63,6 +63,8 @@

     char PVOutputURL[80];           /*--pvouturl     -url   */

     char PVOutputKey[80];           /*--pvoutkey     -key   */

     char PVOutputSid[20];           /*--pvoutsid     -sid   */

+    int PVRateLimit;                /*--pvoutrate    -rate  */

+    int PVBatchSize;                /*--pvoutbatch   -batch */

     char Setting[80];               /* inverter model data  */

     unsigned char InverterCode[4];  /* Unknown code inverter specific*/

     unsigned int ArchiveCode;       /* Code for archive data */

@@ -699,7 +701,9 @@

     //Check if all PVOutput variables are set

     if(( strlen(conf->PVOutputURL) > 0 )

      &&( strlen(conf->PVOutputKey) > 0 )

-     &&( strlen(conf->PVOutputSid) > 0 ))

+     &&( strlen(conf->PVOutputSid) > 0 )

+     &&( conf->PVRateLimit > 0 )

+     &&( conf->PVBatchSize > 0 ))

         (*post)=1;

     else

         (*post)=0;

@@ -774,6 +778,8 @@

     strcpy( conf->PVOutputURL, "http://pvoutput.org/service/r2/addstatus.jsp" );  

     strcpy( conf->PVOutputKey, "" );  

     strcpy( conf->PVOutputSid, "" );

+    conf->PVRateLimit=60;

+    conf->PVBatchSize=30;

     conf->InverterCode[0]=0;

     conf->InverterCode[1]=0;

     conf->InverterCode[2]=0;

@@ -845,6 +851,10 @@

                        strcpy( conf->PVOutputKey, value );  

                     if( strcmp( variable, "PVOutputSid" ) == 0 )

                        strcpy( conf->PVOutputSid, value );  

+                    if( strcmp( variable, "PVRateLimit" ) == 0 )

+                       conf->PVRateLimit =  atoi(value);  

+                    if( strcmp( variable, "PVBatchSize" ) == 0 )

+                       conf->PVBatchSize =  atoi(value);  

                 }

             }

         }

@@ -983,9 +993,11 @@

     printf( "       --INSTALL                           install mysql data tables\n");

     printf( "       --UPDATE                            update mysql data tables\n");

     printf( "PVOutput.org (A free solar information system) Configs\n" );

-    printf( "  -url,  --pvouturl PVOUTURL               pvoutput.org live url\n");

-    printf( "  -key,  --pvoutkey PVOUTKEY               pvoutput.org key\n");

-    printf( "  -sid,  --pvoutsid PVOUTSID               pvoutput.org sid\n");

+    printf( "  -url,    --pvouturl PVOUTURL             pvoutput.org live url\n");

+    printf( "  -key,    --pvoutkey PVOUTKEY             pvoutput.org key\n");

+    printf( "  -sid,    --pvoutsid PVOUTSID             pvoutput.org sid\n");

+    printf( "  -rate,   --pvoutrate PVOUTRATE           pvoutput.org rate limit\n");

+    printf( "  -batch,  --pvoutbatch PVOUTBATCH         pvoutput.org batch limit\n");

     printf( "  -repost                                  verify and repost data if different\n");

     printf( "\n\n" );

 }

@@ -1114,6 +1126,18 @@

                 strcpy(conf->PVOutputSid,argv[i]);

             }

         }

+        else if ((strcmp(argv[i],"-rate")==0)||(strcmp(argv[i],"--pvoutrate")==0)) {

+            i++;

+            if(i<argc){

+                conf->PVRateLimit = atoi(argv[i]);

+            }

+        }

+        else if ((strcmp(argv[i],"-batch")==0)||(strcmp(argv[i],"--pvoutbatch")==0)) {

+            i++;

+            if(i<argc){

+                conf->PVBatchSize = atoi(argv[i]);

+            }

+        }

         else if ((strcmp(argv[i],"-h")==0) || (strcmp(argv[i],"--help") == 0 )) {

             PrintHelp();

             return( -1 );

@@ -1184,7 +1208,7 @@

   return -1;

 }

 

-void post_interval_data(char *pvOutputUrl, char *pvOutputKey, char *pvOutputSid, int repost, char *datefrom, char *dateto, loglevel_t loglevel)

+void post_interval_data(char *pvOutputUrl, char *pvOutputKey, char *pvOutputSid, int PVRateLimit, int PVBatchSize, int repost, char *datefrom, char *dateto, loglevel_t loglevel)

 {

   time_t prior = time(NULL) - ( 60 * 60 * 24 * 14 ); //up to 14 days before now (r2 service)

   struct tm from_datetime = *(localtime( &prior ) );

@@ -1216,9 +1240,12 @@

   struct tm start_datetime, this_datetime;

   int more_rows = 1;

   int string_end = 0;

-  char posturl[2048];

+  int urlmaxlen = 2048;

+  char posturl[urlmaxlen];

   long startOfDayWh = 0;

   int curlResult = -1;

+  int curlHourCount = 0;

+  int curlPostTotal = 0;

   while( more_rows )

   {

     if( 0 == rows_processed )

@@ -1237,22 +1264,37 @@

     rows_processed++;

 

     more_rows = db_row_next( row ); //db_next_row returns 0 if we cannot move to next row in result set, 1 otherwise

-    //r2 service - can process upto 30 rows at a time

-    if( 30 == rows_processed || 0 == more_rows  )

+    //r2 service - can process a maximum rows at a time

+    if( ( PVBatchSize <= rows_processed ) ||

+        ( PVRateLimit <= ( curlHourCount + rows_processed )) ||

+        ( 0 == more_rows ) ||

+        ( 50 >= ( urlmaxlen - strlen( posturl ))) )

     {

       //if post requires last ; to be stripped... posturl[string_end] = '\0';

       curlResult = curl_post_this_query(posturl, pvOutputKey, pvOutputSid, loglevel);

+      curlHourCount = curlHourCount + rows_processed;

+      curlPostTotal = curlPostTotal + rows_processed;

+

       if ( curlResult == 0 )

       {

-        db_set_data_posted(&start_datetime, &this_datetime );  //date range covering possibly 1, but at most 30, values

+        db_set_data_posted(&start_datetime, &this_datetime );  //date range covering possibly 1, but at most PVBatchSize values

         rows_processed = 0;

         sleep(2); //pvoutput api says we can't post more than once a second.

       } else {

           log_error( "CURL post failed, CURL result was %d",curlResult );

       }

+

+      if (( PVRateLimit <= curlHourCount ) && ( 1 == more_rows ))

+      {

+        log_info( "%d API requests made = PVOutput limit. Sleeping for 1 hour..." , curlHourCount);

+        curlHourCount = 0;

+        sleep(3600); // Sleep for 60 mins

+      }

     }

   }

 

+  log_info( "Total %d data points uploaded." , curlPostTotal);

+

   db_row_handle_free( row ); 

 }

 

@@ -2133,7 +2175,7 @@

     archdatalen=0;

     free(last_sent);

     if ((post ==1)&&(mysql==1)&&(error==0)){

-      post_interval_data( conf.PVOutputURL, conf.PVOutputKey, conf.PVOutputSid, repost, datefrom, dateto, loglevel);

+      post_interval_data( conf.PVOutputURL, conf.PVOutputKey, conf.PVOutputSid, conf.PVRateLimit, conf.PVBatchSize, repost, datefrom, dateto, loglevel);

     }

 

 }

[Lem]

This looks interesting. Actually this rate-limiting feature would be great with a "post-only" mode. It would be really cool if smatool could just post to pvoutput without the inverted being contactable. The scenario for me at the moment is that I've moved from my house, but periodically go back there to fetch the inverter data. I then need to post that to pvoutput somehow (preferably not manually)..