Random "Retry policy did not allow for a retry" errors when calling ingestclient.ingest_from_dataframe

Code Sample, a copy-pastable example if possible

def create_df(data):
    print('Constructing df: records = ' + str(len(data)))
    dt = []
    for idx, val in enumerate(data, 0):
        dt.append([idx, json.dumps(val)])

    fields = ['id', 'doc']
    df = pd.DataFrame(data=dt, columns=fields)
    
    return df

def send_data(data, db_name, table_name):
    df = create_df(data)
    
    df.to_csv(table_name + ".csv", index=False, encoding="utf-8", header=False)
    ingestion_props = IngestionProperties(
        database=db_name,
        table=table_name,
        dataFormat=DataFormat.csv,
        # incase status update for success are also required
        # reportLevel=ReportLevel.FailuresAndSuccesses,
        )
    failed = True #assume it will fail
    print("------------------------------------------------------------------------")
    print('Sending to kusto...')
    while failed:
        try:
            ingestclient.ingest_from_dataframe(df, ingestion_properties=ingestion_props)
            failed = False
        except:
            e = traceback.format_exc()
            print(e)
            
            failed = True
            time.sleep(10)

    print('Done sending')
    return

 r = QueryCosmosDbNoonsForClient(clientName)
 send_data(r, dbname, 'noons')
        
 r = GetAllPassagesFromPassageCache(clientName)
 send_data(r, dbname, 'passages')
        
 r = GetAlerts(clientName)
 send_data(r, dbname, 'alerts')
        
 r = GetVesselsWithModules(clientName)
 send_data(r, dbname, 'vesselswithmodules')
        
 r = GetClientList()
 send_data(r, dbname, 'clients')
        
 r = GetAllVesselGroupsForAllUsers(clientName)
        send_data(r, dbname, 'vesselgroupsforusers')
        
r = GetAllVesselsForAClient(clientName)
send_data(r, dbname, 'vessels')

r in each case is just a list of JSON strings.

Problem description

I get random errors when calling ingestclient.ingest_from_dataframe() Example shows multiple calls to the above function and the above function traps for the exception and just retries. The retries are usually successful on the 1st or 2nd attempt. Here is example output:

Constructing df: records = 4610
------------------------------------------------------------------------
Sending to kusto...
Done sending
Constructing df: records = 16
------------------------------------------------------------------------
Sending to kusto...
Done sending
Constructing df: records = 287
------------------------------------------------------------------------
Sending to kusto...
------------------------------------------------------------------------
Sending to kusto...
ERROR:azure.storage.common.storageclient:Client-Request-ID=025eb4ca-e524-11e9-8b7e-70886b83b7da Retry policy did not allow for a retry: Server-Timestamp=Wed, 02 Oct 2019 14:50:42 GMT, Server-Request-ID=887414da-8003-000b-2830-7984e0000000, HTTP status code=400, Exception=The value for one of the HTTP headers is not in the correct format.<?xml version="1.0" encoding="utf-8"?><Error><Code>InvalidHeaderValue</Code><Message>The value for one of the HTTP headers is not in the correct format.RequestId:887414da-8003-000b-2830-7984e0000000Time:2019-10-02T14:50:43.4555251Z</Message><HeaderName>x-ms-version</HeaderName><HeaderValue>2019-02-02</HeaderValue></Error>.
Traceback (most recent call last):
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\KustoCache\__init__.py", line 108, in send_data
    ingestclient.ingest_from_dataframe(df, ingestion_properties=ingestion_props)
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\kusto\ingest\_ingest_client.py", line 66, in ingest_from_dataframe
    self.ingest_from_blob(BlobDescriptor(url, fd.size), ingestion_properties=ingestion_properties)
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\kusto\ingest\_ingest_client.py", line 121, in ingest_from_blob
    queue_service.put_message(queue_name=queue_details.object_name, content=encoded)
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\storage\queue\queueservice.py", line 793, in put_message
    None, None, content])
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\storage\common\storageclient.py", line 430, in _perform_request
    raise ex
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\storage\common\storageclient.py", line 358, in _perform_request
    raise ex
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\storage\common\storageclient.py", line 344, in _perform_request
    HTTPError(response.status, response.message, response.headers, response.body))
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\storage\common\_error.py", line 115, in _http_error_handler
    raise ex
azure.common.AzureHttpError: The value for one of the HTTP headers is not in the correct format.
<?xml version="1.0" encoding="utf-8"?><Error><Code>InvalidHeaderValue</Code><Message>The value for one of the HTTP headers is not in the correct format.
RequestId:887414da-8003-000b-2830-7984e0000000
Time:2019-10-02T14:50:43.4555251Z</Message><HeaderName>x-ms-version</HeaderName><HeaderValue>2019-02-02</HeaderValue></Error>

Done sending
Constructing df: records = 4
------------------------------------------------------------------------
Sending to kusto...
Done sending
Constructing df: records = 20
------------------------------------------------------------------------
Sending to kusto...
Done sending
Constructing df: records = 16
------------------------------------------------------------------------
Sending to kusto...
Done sending
Constructing df: records = 2
------------------------------------------------------------------------
Sending to kusto...
ERROR:azure.storage.common.storageclient:Client-Request-ID=0d71a462-e524-11e9-8a6a-70886b83b7da Retry policy did not allow for a retry: Server-Timestamp=Wed, 02 Oct 2019 14:51:01 GMT, Server-Request-ID=467a7e1b-2003-0060-4b30-790314000000, HTTP status code=400, Exception=The value for one of the HTTP headers is not in the correct format.<?xml version="1.0" encoding="utf-8"?><Error><Code>InvalidHeaderValue</Code><Message>The value for one of the HTTP headers is not in the correct format.RequestId:467a7e1b-2003-0060-4b30-790314000000Time:2019-10-02T14:51:01.9837789Z</Message><HeaderName>x-ms-version</HeaderName><HeaderValue>2019-02-02</HeaderValue></Error>.
Traceback (most recent call last):
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\KustoCache\__init__.py", line 108, in send_data
    ingestclient.ingest_from_dataframe(df, ingestion_properties=ingestion_props)
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\kusto\ingest\_ingest_client.py", line 66, in ingest_from_dataframe
    self.ingest_from_blob(BlobDescriptor(url, fd.size), ingestion_properties=ingestion_properties)
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\kusto\ingest\_ingest_client.py", line 121, in ingest_from_blob
    queue_service.put_message(queue_name=queue_details.object_name, content=encoded)
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\storage\queue\queueservice.py", line 793, in put_message
    None, None, content])
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\storage\common\storageclient.py", line 430, in _perform_request
    raise ex
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\storage\common\storageclient.py", line 358, in _perform_request
    raise ex
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\storage\common\storageclient.py", line 344, in _perform_request
    HTTPError(response.status, response.message, response.headers, response.body))
  File "c:\dev\src\i4\Services\i4ServicesPyParsing\.env\lib\site-packages\azure\storage\common\_error.py", line 115, in _http_error_handler
    raise ex
azure.common.AzureHttpError: The value for one of the HTTP headers is not in the correct format.
<?xml version="1.0" encoding="utf-8"?><Error><Code>InvalidHeaderValue</Code><Message>The value for one of the HTTP headers is not in the correct format.
RequestId:467a7e1b-2003-0060-4b30-790314000000
Time:2019-10-02T14:51:01.9837789Z</Message><HeaderName>x-ms-version</HeaderName><HeaderValue>2019-02-02</HeaderValue></Error>

Done sending

Output of pip freeze

[paste the output of pip freeze here below this line] adal==1.2.1 antlr4-python3-runtime==4.7.2 applicationinsights==0.11.7
argcomplete==1.10.0 asn1crypto==0.24.0 astroid==2.1.0 atomicwrites==1.3.0 attrs==19.1.0 azure-batch==6.0.0 azure-cli==2.0.67 azure-cli-acr==2.2.9 azure-cli-acs==2.4.4 azure-cli-advisor==2.0.1 azure-cli-ams==0.4.7 azure-cli-appservice==0.2.21 azure-cli-backup==1.2.5 azure-cli-batch==4.0.3 azure-cli-batchai==0.4.10 azure-cli-billing==0.2.2 azure-cli-botservice==0.2.2 azure-cli-cdn==0.2.4 azure-cli-cloud==2.1.1 azure-cli-cognitiveservices==0.2.6 azure-cli-command-modules-nspkg==2.0.2 azure-cli-configure==2.0.24 azure-cli-consumption==0.4.4 azure-cli-container==0.3.18 azure-cli-core==2.0.67 azure-cli-cosmosdb==0.2.11 azure-cli-deploymentmanager==0.1.1 azure-cli-dla==0.2.6 azure-cli-dls==0.1.10 azure-cli-dms==0.1.4 azure-cli-eventgrid==0.2.4 azure-cli-eventhubs==0.3.7 azure-cli-extension==0.2.5 azure-cli-feedback==2.2.1 azure-cli-find==0.3.4 azure-cli-hdinsight==0.3.5 azure-cli-interactive==0.4.5 azure-cli-iot==0.3.11 azure-cli-iotcentral==0.1.7 azure-cli-keyvault==2.2.16 azure-cli-kusto==0.2.3 azure-cli-lab==0.1.8 azure-cli-maps==0.3.5 azure-cli-monitor==0.2.15 azure-cli-natgateway==0.1.1 azure-cli-network==2.5.2 azure-cli-nspkg==3.0.3 azure-cli-policyinsights==0.1.4 azure-cli-privatedns==1.0.2 azure-cli-profile==2.1.5 azure-cli-rdbms==0.3.12 azure-cli-redis==0.4.4 azure-cli-relay==0.1.5 azure-cli-reservations==0.4.3 azure-cli-resource==2.1.16 azure-cli-role==2.6.4 azure-cli-search==0.1.2 azure-cli-security==0.1.2 azure-cli-servicebus==0.3.6 azure-cli-servicefabric==0.1.20 azure-cli-signalr==1.0.1 azure-cli-sql==2.2.5 azure-cli-sqlvm==0.2.0 azure-cli-storage==2.4.3 azure-cli-telemetry==1.0.2 azure-cli-vm==2.2.23 azure-common==1.1.23 azure-cosmos==3.0.2 azure-datalake-store==0.0.39 azure-functions==1.0.0b5 azure-functions-devops-build==0.0.22 azure-functions-worker==1.0.0b10 azure-graphrbac==0.60.0 azure-keyvault==1.1.0 azure-kusto-data==0.0.33 azure-kusto-ingest==0.0.33 azure-mgmt-advisor==2.0.1 azure-mgmt-applicationinsights==0.1.1 azure-mgmt-authorization==0.50.0 azure-mgmt-batch==6.0.0 azure-mgmt-batchai==2.0.0 azure-mgmt-billing==0.2.0 azure-mgmt-botservice==0.2.0 azure-mgmt-cdn==3.1.0 azure-mgmt-cognitiveservices==3.0.0 azure-mgmt-compute==5.0.0 azure-mgmt-consumption==2.0.0 azure-mgmt-containerinstance==1.4.0 azure-mgmt-containerregistry==2.8.0 azure-mgmt-containerservice==5.2.0 azure-mgmt-cosmosdb==0.6.1 azure-mgmt-datalake-analytics==0.2.1 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-datamigration==0.1.0 azure-mgmt-deploymentmanager==0.1.0 azure-mgmt-devtestlabs==2.2.0 azure-mgmt-dns==2.1.0 azure-mgmt-eventgrid==2.2.0 azure-mgmt-eventhub==2.6.0 azure-mgmt-hdinsight==0.2.1 azure-mgmt-imagebuilder==0.2.1 azure-mgmt-iotcentral==1.0.0 azure-mgmt-iothub==0.8.2 azure-mgmt-iothubprovisioningservices==0.2.0 azure-mgmt-keyvault==1.1.0 azure-mgmt-kusto==0.3.0 azure-mgmt-loganalytics==0.2.0 azure-mgmt-managementgroups==0.1.0 azure-mgmt-maps==0.1.0 azure-mgmt-marketplaceordering==0.1.0 azure-mgmt-media==1.1.1 azure-mgmt-monitor==0.5.2 azure-mgmt-msi==0.2.0 azure-mgmt-network==3.0.0 azure-mgmt-nspkg==3.0.2 azure-mgmt-policyinsights==0.3.1 azure-mgmt-privatedns==0.1.0 azure-mgmt-rdbms==1.8.0 azure-mgmt-recoveryservices==0.1.1 azure-mgmt-recoveryservicesbackup==0.1.2 azure-mgmt-redis==6.0.0 azure-mgmt-relay==0.1.0 azure-mgmt-reservations==0.3.1 azure-mgmt-resource==2.1.0 azure-mgmt-search==2.0.0 azure-mgmt-security==0.1.0 azure-mgmt-servicebus==0.6.0 azure-mgmt-servicefabric==0.2.0 azure-mgmt-signalr==0.1.1 azure-mgmt-sql==0.12.0 azure-mgmt-sqlvirtualmachine==0.3.0 azure-mgmt-storage==3.3.0 azure-mgmt-trafficmanager==0.51.0 azure-mgmt-web==0.42.0 azure-multiapi-storage==0.2.3 azure-nspkg==3.0.2 azure-storage-blob==1.3.1 azure-storage-common==1.4.2 azure-storage-nspkg==3.1.0 azure-storage-queue==2.1.0 bcrypt==3.1.7 certifi==2018.11.29 cffi==1.12.3 cftime==1.0.3.4 chardet==3.0.4 colorama==0.4.1 cryptography==2.7 DateTimeRange==0.5.5 fabric==2.4.0 geographiclib==1.49 grpcio==1.14.2 grpcio-tools==1.14.2 html2text==2018.1.9 humanfriendly==4.18 idna==2.8 importlib-metadata==0.20 invoke==1.2.0 ipaddress==1.0.22 isodate==0.6.0 isort==4.3.4 Jinja2==2.10.1 jmespath==0.9.4 knack==0.6.2 lazy-object-proxy==1.3.1 mail-parser==3.8.1 MarkupSafe==1.1.1 mbstrdecoder==0.7.0 mccabe==0.6.1 mock==3.0.5 more-itertools==7.2.0 msrest==0.6.8 msrestazure==0.6.1 netCDF4==1.4.2 numpy==1.16.0 oauthlib==3.0.1 packaging==19.1 pandas==0.24.0 paramiko==2.6.0 pluggy==0.12.0 portalocker==1.2.1 prompt-toolkit==1.0.16 protobuf==3.6.1 psutil==5.6.3 ptvsd==4.2.2 py==1.8.0 pycparser==2.19 Pygments==2.4.2 PyJWT==1.7.1 pylint==2.2.2 PyNaCl==1.3.0 pyOpenSSL==19.0.0 pyparsing==2.4.2 pyperclip==1.7.0 pypiwin32==223 pyreadline==2.1 pytest==5.1.2 python-dateutil==2.8.0 pytz==2018.9 pywin32==224 PyYAML==5.1.1 requests==2.21.0 requests-oauthlib==1.2.0 scipy==1.2.0 scp==0.13.2 simplejson==3.16.0 six==1.12.0 sshtunnel==0.1.5 tabulate==0.8.3 typed-ast==1.2.0 typepy==0.4.0 urllib3==1.24.1 vsts==0.1.25 vsts-cd-manager==1.0.2 wcwidth==0.1.7 websocket-client==0.56.0 wrapt==1.11.1 xarray==0.11.3 xlrd==1.2.0 xmltodict==0.12.0 zipp==0.6.0

[danield137]

Hi @docgregt ,You are correct, we should document that the enums changed. @boazsha can you do that please?

Regarding reserved words, that is not an sdk problem, but you can always escape reserved words by wrapping with brackets:

['consumers'] | count should do the trick. do let us know if you run into more trouble.

[docgregt]

One item I ran into with the .38 version is that ‘consumers’ seems to be a reserved word now. We had a Kusto table called ‘consumers’ but could not delete it from the python sdk any longer.