venv added, updated
This commit is contained in:
Norbert
@@ -0,0 +1,56 @@
|
||||
# flake8: noqa
|
||||
|
||||
"""
|
||||
InfluxDB OSS API Service.
|
||||
|
||||
The InfluxDB v2 API provides a programmatic interface for all interactions with InfluxDB. Access the InfluxDB API using the `/api/v2/` endpoint. # noqa: E501
|
||||
|
||||
OpenAPI spec version: 2.0.0
|
||||
Generated by: https://openapi-generator.tech
|
||||
"""
|
||||
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
# import apis into api package
|
||||
from influxdb_client.service.authorizations_service import AuthorizationsService
|
||||
from influxdb_client.service.backup_service import BackupService
|
||||
from influxdb_client.service.bucket_schemas_service import BucketSchemasService
|
||||
from influxdb_client.service.buckets_service import BucketsService
|
||||
from influxdb_client.service.cells_service import CellsService
|
||||
from influxdb_client.service.checks_service import ChecksService
|
||||
from influxdb_client.service.config_service import ConfigService
|
||||
from influxdb_client.service.dbr_ps_service import DBRPsService
|
||||
from influxdb_client.service.dashboards_service import DashboardsService
|
||||
from influxdb_client.service.delete_service import DeleteService
|
||||
from influxdb_client.service.health_service import HealthService
|
||||
from influxdb_client.service.invokable_scripts_service import InvokableScriptsService
|
||||
from influxdb_client.service.labels_service import LabelsService
|
||||
from influxdb_client.service.legacy_authorizations_service import LegacyAuthorizationsService
|
||||
from influxdb_client.service.metrics_service import MetricsService
|
||||
from influxdb_client.service.notification_endpoints_service import NotificationEndpointsService
|
||||
from influxdb_client.service.notification_rules_service import NotificationRulesService
|
||||
from influxdb_client.service.organizations_service import OrganizationsService
|
||||
from influxdb_client.service.ping_service import PingService
|
||||
from influxdb_client.service.query_service import QueryService
|
||||
from influxdb_client.service.ready_service import ReadyService
|
||||
from influxdb_client.service.remote_connections_service import RemoteConnectionsService
|
||||
from influxdb_client.service.replications_service import ReplicationsService
|
||||
from influxdb_client.service.resources_service import ResourcesService
|
||||
from influxdb_client.service.restore_service import RestoreService
|
||||
from influxdb_client.service.routes_service import RoutesService
|
||||
from influxdb_client.service.rules_service import RulesService
|
||||
from influxdb_client.service.scraper_targets_service import ScraperTargetsService
|
||||
from influxdb_client.service.secrets_service import SecretsService
|
||||
from influxdb_client.service.setup_service import SetupService
|
||||
from influxdb_client.service.signin_service import SigninService
|
||||
from influxdb_client.service.signout_service import SignoutService
|
||||
from influxdb_client.service.sources_service import SourcesService
|
||||
from influxdb_client.service.tasks_service import TasksService
|
||||
from influxdb_client.service.telegraf_plugins_service import TelegrafPluginsService
|
||||
from influxdb_client.service.telegrafs_service import TelegrafsService
|
||||
from influxdb_client.service.templates_service import TemplatesService
|
||||
from influxdb_client.service.users_service import UsersService
|
||||
from influxdb_client.service.variables_service import VariablesService
|
||||
from influxdb_client.service.views_service import ViewsService
|
||||
from influxdb_client.service.write_service import WriteService
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,290 @@
|
||||
"""
|
||||
Functions for serialize Pandas DataFrame.
|
||||
|
||||
Much of the code here is inspired by that in the aioinflux packet found here: https://github.com/gusutabopb/aioinflux
|
||||
"""
|
||||
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
|
||||
from influxdb_client import WritePrecision
|
||||
from influxdb_client.client.write.point import _ESCAPE_KEY, _ESCAPE_STRING, _ESCAPE_MEASUREMENT, DEFAULT_WRITE_PRECISION
|
||||
|
||||
logger = logging.getLogger('influxdb_client.client.write.dataframe_serializer')
|
||||
|
||||
|
||||
def _itertuples(data_frame):
|
||||
cols = [data_frame.iloc[:, k] for k in range(len(data_frame.columns))]
|
||||
return zip(data_frame.index, *cols)
|
||||
|
||||
|
||||
class DataframeSerializer:
|
||||
"""Serialize DataFrame into LineProtocols."""
|
||||
|
||||
def __init__(self, data_frame, point_settings, precision=DEFAULT_WRITE_PRECISION, chunk_size: int = None,
|
||||
**kwargs) -> None:
|
||||
"""
|
||||
Init serializer.
|
||||
|
||||
:param data_frame: Pandas DataFrame to serialize
|
||||
:param point_settings: Default Tags
|
||||
:param precision: The precision for the unix timestamps within the body line-protocol.
|
||||
:param chunk_size: The size of chunk for serializing into chunks.
|
||||
:key data_frame_measurement_name: name of measurement for writing Pandas DataFrame
|
||||
:key data_frame_tag_columns: list of DataFrame columns which are tags, rest columns will be fields
|
||||
:key data_frame_timestamp_column: name of DataFrame column which contains a timestamp. The column can be defined as a :class:`~str` value
|
||||
formatted as `2018-10-26`, `2018-10-26 12:00`, `2018-10-26 12:00:00-05:00`
|
||||
or other formats and types supported by `pandas.to_datetime <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.to_datetime.html#pandas.to_datetime>`_ - ``DataFrame``
|
||||
:key data_frame_timestamp_timezone: name of the timezone which is used for timestamp column - ``DataFrame``
|
||||
""" # noqa: E501
|
||||
# This function is hard to understand but for good reason:
|
||||
# the approach used here is considerably more efficient
|
||||
# than the alternatives.
|
||||
#
|
||||
# We build up a Python expression that efficiently converts a data point
|
||||
# tuple into line-protocol entry, and then evaluate the expression
|
||||
# as a lambda so that we can call it. This avoids the overhead of
|
||||
# invoking a function on every data value - we only have one function
|
||||
# call per row instead. The expression consists of exactly
|
||||
# one f-string, so we build up the parts of it as segments
|
||||
# that are concatenated together to make the full f-string inside
|
||||
# the lambda.
|
||||
#
|
||||
# Things are made a little more complex because fields and tags with NaN
|
||||
# values and empty tags are omitted from the generated line-protocol
|
||||
# output.
|
||||
#
|
||||
# As an example, say we have a data frame with two value columns:
|
||||
# a float
|
||||
# b int
|
||||
#
|
||||
# This will generate a lambda expression to be evaluated that looks like
|
||||
# this:
|
||||
#
|
||||
# lambda p: f"""{measurement_name} {keys[0]}={p[1]},{keys[1]}={p[2]}i {p[0].value}"""
|
||||
#
|
||||
# This lambda is then executed for each row p.
|
||||
#
|
||||
# When NaNs are present, the expression looks like this (split
|
||||
# across two lines to satisfy the code-style checker)
|
||||
#
|
||||
# lambda p: f"""{measurement_name} {"" if pd.isna(p[1])
|
||||
# else f"{keys[0]}={p[1]}"},{keys[1]}={p[2]}i {p[0].value}"""
|
||||
#
|
||||
# When there's a NaN value in column a, we'll end up with a comma at the start of the
|
||||
# fields, so we run a regexp substitution after generating the line-protocol entries
|
||||
# to remove this.
|
||||
#
|
||||
# We're careful to run these potentially costly extra steps only when NaN values actually
|
||||
# exist in the data.
|
||||
|
||||
from ...extras import pd, np
|
||||
if not isinstance(data_frame, pd.DataFrame):
|
||||
raise TypeError('Must be DataFrame, but type was: {0}.'
|
||||
.format(type(data_frame)))
|
||||
|
||||
data_frame_measurement_name = kwargs.get('data_frame_measurement_name')
|
||||
if data_frame_measurement_name is None:
|
||||
raise TypeError('"data_frame_measurement_name" is a Required Argument')
|
||||
|
||||
timestamp_column = kwargs.get('data_frame_timestamp_column', None)
|
||||
timestamp_timezone = kwargs.get('data_frame_timestamp_timezone', None)
|
||||
data_frame = data_frame.copy(deep=False)
|
||||
data_frame_timestamp = data_frame.index if timestamp_column is None else data_frame[timestamp_column]
|
||||
if isinstance(data_frame_timestamp, pd.PeriodIndex):
|
||||
data_frame_timestamp = data_frame_timestamp.to_timestamp()
|
||||
else:
|
||||
# TODO: this is almost certainly not what you want
|
||||
# when the index is the default RangeIndex.
|
||||
# Instead, it would probably be better to leave
|
||||
# out the timestamp unless a time column is explicitly
|
||||
# enabled.
|
||||
data_frame_timestamp = pd.to_datetime(data_frame_timestamp, unit=precision)
|
||||
|
||||
if timestamp_timezone:
|
||||
if isinstance(data_frame_timestamp, pd.DatetimeIndex):
|
||||
data_frame_timestamp = data_frame_timestamp.tz_localize(timestamp_timezone)
|
||||
else:
|
||||
data_frame_timestamp = data_frame_timestamp.dt.tz_localize(timestamp_timezone)
|
||||
|
||||
if hasattr(data_frame_timestamp, 'tzinfo') and data_frame_timestamp.tzinfo is None:
|
||||
data_frame_timestamp = data_frame_timestamp.tz_localize('UTC')
|
||||
if timestamp_column is None:
|
||||
data_frame.index = data_frame_timestamp
|
||||
else:
|
||||
data_frame[timestamp_column] = data_frame_timestamp
|
||||
|
||||
data_frame_tag_columns = kwargs.get('data_frame_tag_columns')
|
||||
data_frame_tag_columns = set(data_frame_tag_columns or [])
|
||||
|
||||
# keys holds a list of string keys.
|
||||
keys = []
|
||||
# tags holds a list of tag f-string segments ordered alphabetically by tag key.
|
||||
tags = []
|
||||
# fields holds a list of field f-string segments ordered alphebetically by field key
|
||||
fields = []
|
||||
# field_indexes holds the index into each row of all the fields.
|
||||
field_indexes = []
|
||||
|
||||
if point_settings.defaultTags:
|
||||
for key, value in point_settings.defaultTags.items():
|
||||
# Avoid overwriting existing data if there's a column
|
||||
# that already exists with the default tag's name.
|
||||
# Note: when a new column is added, the old DataFrame
|
||||
# that we've made a shallow copy of is unaffected.
|
||||
# TODO: when there are NaN or empty values in
|
||||
# the column, we could make a deep copy of the
|
||||
# data and fill in those values with the default tag value.
|
||||
if key not in data_frame.columns:
|
||||
data_frame[key] = value
|
||||
data_frame_tag_columns.add(key)
|
||||
|
||||
# Get a list of all the columns sorted by field/tag key.
|
||||
# We want to iterate through the columns in sorted order
|
||||
# so that we know when we're on the first field so we
|
||||
# can know whether a comma is needed for that
|
||||
# field.
|
||||
columns = sorted(enumerate(data_frame.dtypes.items()), key=lambda col: col[1][0])
|
||||
|
||||
# null_columns has a bool value for each column holding
|
||||
# whether that column contains any null (NaN or None) values.
|
||||
null_columns = data_frame.isnull().any()
|
||||
timestamp_index = 0
|
||||
|
||||
# Iterate through the columns building up the expression for each column.
|
||||
for index, (key, value) in columns:
|
||||
key = str(key)
|
||||
key_format = f'{{keys[{len(keys)}]}}'
|
||||
keys.append(key.translate(_ESCAPE_KEY))
|
||||
# The field index is one more than the column index because the
|
||||
# time index is at column zero in the finally zipped-together
|
||||
# result columns.
|
||||
field_index = index + 1
|
||||
val_format = f'p[{field_index}]'
|
||||
|
||||
if key in data_frame_tag_columns:
|
||||
# This column is a tag column.
|
||||
if null_columns.iloc[index]:
|
||||
key_value = f"""{{
|
||||
'' if {val_format} == '' or pd.isna({val_format}) else
|
||||
f',{key_format}={{str({val_format}).translate(_ESCAPE_STRING)}}'
|
||||
}}"""
|
||||
else:
|
||||
key_value = f',{key_format}={{str({val_format}).translate(_ESCAPE_KEY)}}'
|
||||
tags.append(key_value)
|
||||
continue
|
||||
elif timestamp_column is not None and key in timestamp_column:
|
||||
timestamp_index = field_index
|
||||
continue
|
||||
|
||||
# This column is a field column.
|
||||
# Note: no comma separator is needed for the first field.
|
||||
# It's important to omit it because when the first
|
||||
# field column has no nulls, we don't run the comma-removal
|
||||
# regexp substitution step.
|
||||
sep = '' if len(field_indexes) == 0 else ','
|
||||
if issubclass(value.type, np.integer) or issubclass(value.type, np.floating) or issubclass(value.type, np.bool_): # noqa: E501
|
||||
suffix = 'i' if issubclass(value.type, np.integer) else ''
|
||||
if null_columns.iloc[index]:
|
||||
field_value = f"""{{"" if pd.isna({val_format}) else f"{sep}{key_format}={{{val_format}}}{suffix}"}}""" # noqa: E501
|
||||
else:
|
||||
field_value = f"{sep}{key_format}={{{val_format}}}{suffix}"
|
||||
else:
|
||||
if null_columns.iloc[index]:
|
||||
field_value = f"""{{
|
||||
'' if pd.isna({val_format}) else
|
||||
f'{sep}{key_format}="{{str({val_format}).translate(_ESCAPE_STRING)}}"'
|
||||
}}"""
|
||||
else:
|
||||
field_value = f'''{sep}{key_format}="{{str({val_format}).translate(_ESCAPE_STRING)}}"'''
|
||||
field_indexes.append(field_index)
|
||||
fields.append(field_value)
|
||||
|
||||
measurement_name = str(data_frame_measurement_name).translate(_ESCAPE_MEASUREMENT)
|
||||
|
||||
tags = ''.join(tags)
|
||||
fields = ''.join(fields)
|
||||
timestamp = '{p[%s].value}' % timestamp_index
|
||||
if precision == WritePrecision.US:
|
||||
timestamp = '{int(p[%s].value / 1e3)}' % timestamp_index
|
||||
elif precision == WritePrecision.MS:
|
||||
timestamp = '{int(p[%s].value / 1e6)}' % timestamp_index
|
||||
elif precision == WritePrecision.S:
|
||||
timestamp = '{int(p[%s].value / 1e9)}' % timestamp_index
|
||||
|
||||
f = eval(f'lambda p: f"""{{measurement_name}}{tags} {fields} {timestamp}"""', {
|
||||
'measurement_name': measurement_name,
|
||||
'_ESCAPE_KEY': _ESCAPE_KEY,
|
||||
'_ESCAPE_STRING': _ESCAPE_STRING,
|
||||
'keys': keys,
|
||||
'pd': pd,
|
||||
})
|
||||
|
||||
for k, v in dict(data_frame.dtypes).items():
|
||||
if k in data_frame_tag_columns:
|
||||
data_frame = data_frame.replace({k: ''}, np.nan)
|
||||
|
||||
def _any_not_nan(p, indexes):
|
||||
return any(map(lambda x: not pd.isna(p[x]), indexes))
|
||||
|
||||
self.data_frame = data_frame
|
||||
self.f = f
|
||||
self.field_indexes = field_indexes
|
||||
self.first_field_maybe_null = null_columns.iloc[field_indexes[0] - 1]
|
||||
self._any_not_nan = _any_not_nan
|
||||
|
||||
#
|
||||
# prepare chunks
|
||||
#
|
||||
if chunk_size is not None:
|
||||
self.number_of_chunks = int(math.ceil(len(data_frame) / float(chunk_size)))
|
||||
self.chunk_size = chunk_size
|
||||
else:
|
||||
self.number_of_chunks = None
|
||||
|
||||
def serialize(self, chunk_idx: int = None):
|
||||
"""
|
||||
Serialize chunk into LineProtocols.
|
||||
|
||||
:param chunk_idx: The index of chunk to serialize. If `None` then serialize whole dataframe.
|
||||
"""
|
||||
if chunk_idx is None:
|
||||
chunk = self.data_frame
|
||||
else:
|
||||
logger.debug("Serialize chunk %s/%s ...", chunk_idx + 1, self.number_of_chunks)
|
||||
chunk = self.data_frame[chunk_idx * self.chunk_size:(chunk_idx + 1) * self.chunk_size]
|
||||
|
||||
if self.first_field_maybe_null:
|
||||
# When the first field is null (None/NaN), we'll have
|
||||
# a spurious leading comma which needs to be removed.
|
||||
lp = (re.sub('^(( |[^ ])* ),([a-zA-Z0-9])(.*)', '\\1\\3\\4', self.f(p))
|
||||
for p in filter(lambda x: self._any_not_nan(x, self.field_indexes), _itertuples(chunk)))
|
||||
return list(lp)
|
||||
else:
|
||||
return list(map(self.f, _itertuples(chunk)))
|
||||
|
||||
def number_of_chunks(self):
|
||||
"""
|
||||
Return the number of chunks.
|
||||
|
||||
:return: number of chunks or None if chunk_size is not specified.
|
||||
"""
|
||||
return self.number_of_chunks
|
||||
|
||||
|
||||
def data_frame_to_list_of_points(data_frame, point_settings, precision=DEFAULT_WRITE_PRECISION, **kwargs):
|
||||
"""
|
||||
Serialize DataFrame into LineProtocols.
|
||||
|
||||
:param data_frame: Pandas DataFrame to serialize
|
||||
:param point_settings: Default Tags
|
||||
:param precision: The precision for the unix timestamps within the body line-protocol.
|
||||
:key data_frame_measurement_name: name of measurement for writing Pandas DataFrame
|
||||
:key data_frame_tag_columns: list of DataFrame columns which are tags, rest columns will be fields
|
||||
:key data_frame_timestamp_column: name of DataFrame column which contains a timestamp. The column can be defined as a :class:`~str` value
|
||||
formatted as `2018-10-26`, `2018-10-26 12:00`, `2018-10-26 12:00:00-05:00`
|
||||
or other formats and types supported by `pandas.to_datetime <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.to_datetime.html#pandas.to_datetime>`_ - ``DataFrame``
|
||||
:key data_frame_timestamp_timezone: name of the timezone which is used for timestamp column - ``DataFrame``
|
||||
""" # noqa: E501
|
||||
return DataframeSerializer(data_frame, point_settings, precision, **kwargs).serialize()
|
||||
@@ -0,0 +1,371 @@
|
||||
"""Point data structure to represent LineProtocol."""
|
||||
|
||||
import math
|
||||
import warnings
|
||||
from builtins import int
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
from numbers import Integral
|
||||
|
||||
from influxdb_client.client.util.date_utils import get_date_helper
|
||||
from influxdb_client.domain.write_precision import WritePrecision
|
||||
|
||||
EPOCH = datetime.fromtimestamp(0, tz=timezone.utc)
|
||||
|
||||
DEFAULT_WRITE_PRECISION = WritePrecision.NS
|
||||
|
||||
_ESCAPE_MEASUREMENT = str.maketrans({
|
||||
',': r'\,',
|
||||
' ': r'\ ',
|
||||
'\n': r'\n',
|
||||
'\t': r'\t',
|
||||
'\r': r'\r',
|
||||
})
|
||||
|
||||
_ESCAPE_KEY = str.maketrans({
|
||||
',': r'\,',
|
||||
'=': r'\=',
|
||||
' ': r'\ ',
|
||||
'\n': r'\n',
|
||||
'\t': r'\t',
|
||||
'\r': r'\r',
|
||||
})
|
||||
|
||||
_ESCAPE_STRING = str.maketrans({
|
||||
'"': r'\"',
|
||||
'\\': r'\\',
|
||||
})
|
||||
|
||||
try:
|
||||
import numpy as np
|
||||
|
||||
_HAS_NUMPY = True
|
||||
except ModuleNotFoundError:
|
||||
_HAS_NUMPY = False
|
||||
|
||||
|
||||
class Point(object):
|
||||
"""
|
||||
Point defines the values that will be written to the database.
|
||||
|
||||
Ref: https://docs.influxdata.com/influxdb/latest/reference/key-concepts/data-elements/#point
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def measurement(measurement):
|
||||
"""Create a new Point with specified measurement name."""
|
||||
p = Point(measurement)
|
||||
return p
|
||||
|
||||
@staticmethod
|
||||
def from_dict(dictionary: dict, write_precision: WritePrecision = DEFAULT_WRITE_PRECISION, **kwargs):
|
||||
"""
|
||||
Initialize point from 'dict' structure.
|
||||
|
||||
The expected dict structure is:
|
||||
- measurement
|
||||
- tags
|
||||
- fields
|
||||
- time
|
||||
|
||||
Example:
|
||||
.. code-block:: python
|
||||
|
||||
# Use default dictionary structure
|
||||
dict_structure = {
|
||||
"measurement": "h2o_feet",
|
||||
"tags": {"location": "coyote_creek"},
|
||||
"fields": {"water_level": 1.0},
|
||||
"time": 1
|
||||
}
|
||||
point = Point.from_dict(dict_structure, WritePrecision.NS)
|
||||
|
||||
Example:
|
||||
.. code-block:: python
|
||||
|
||||
# Use custom dictionary structure
|
||||
dictionary = {
|
||||
"name": "sensor_pt859",
|
||||
"location": "warehouse_125",
|
||||
"version": "2021.06.05.5874",
|
||||
"pressure": 125,
|
||||
"temperature": 10,
|
||||
"created": 1632208639,
|
||||
}
|
||||
point = Point.from_dict(dictionary,
|
||||
write_precision=WritePrecision.S,
|
||||
record_measurement_key="name",
|
||||
record_time_key="created",
|
||||
record_tag_keys=["location", "version"],
|
||||
record_field_keys=["pressure", "temperature"])
|
||||
|
||||
Int Types:
|
||||
The following example shows how to configure the types of integers fields.
|
||||
It is useful when you want to serialize integers always as ``float`` to avoid ``field type conflict``
|
||||
or use ``unsigned 64-bit integer`` as the type for serialization.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# Use custom dictionary structure
|
||||
dict_structure = {
|
||||
"measurement": "h2o_feet",
|
||||
"tags": {"location": "coyote_creek"},
|
||||
"fields": {
|
||||
"water_level": 1.0,
|
||||
"some_counter": 108913123234
|
||||
},
|
||||
"time": 1
|
||||
}
|
||||
|
||||
point = Point.from_dict(dict_structure, field_types={"some_counter": "uint"})
|
||||
|
||||
:param dictionary: dictionary for serialize into data Point
|
||||
:param write_precision: sets the precision for the supplied time values
|
||||
:key record_measurement_key: key of dictionary with specified measurement
|
||||
:key record_measurement_name: static measurement name for data Point
|
||||
:key record_time_key: key of dictionary with specified timestamp
|
||||
:key record_tag_keys: list of dictionary keys to use as a tag
|
||||
:key record_field_keys: list of dictionary keys to use as a field
|
||||
:key field_types: optional dictionary to specify types of serialized fields. Currently, is supported customization for integer types.
|
||||
Possible integers types:
|
||||
- ``int`` - serialize integers as "**Signed 64-bit integers**" - ``9223372036854775807i`` (default behaviour)
|
||||
- ``uint`` - serialize integers as "**Unsigned 64-bit integers**" - ``9223372036854775807u``
|
||||
- ``float`` - serialize integers as "**IEEE-754 64-bit floating-point numbers**". Useful for unify number types in your pipeline to avoid field type conflict - ``9223372036854775807``
|
||||
The ``field_types`` can be also specified as part of incoming dictionary. For more info see an example above.
|
||||
:return: new data point
|
||||
""" # noqa: E501
|
||||
measurement_ = kwargs.get('record_measurement_name', None)
|
||||
if measurement_ is None:
|
||||
measurement_ = dictionary[kwargs.get('record_measurement_key', 'measurement')]
|
||||
point = Point(measurement_)
|
||||
|
||||
record_tag_keys = kwargs.get('record_tag_keys', None)
|
||||
if record_tag_keys is not None:
|
||||
for tag_key in record_tag_keys:
|
||||
if tag_key in dictionary:
|
||||
point.tag(tag_key, dictionary[tag_key])
|
||||
elif 'tags' in dictionary:
|
||||
for tag_key, tag_value in dictionary['tags'].items():
|
||||
point.tag(tag_key, tag_value)
|
||||
|
||||
record_field_keys = kwargs.get('record_field_keys', None)
|
||||
if record_field_keys is not None:
|
||||
for field_key in record_field_keys:
|
||||
if field_key in dictionary:
|
||||
point.field(field_key, dictionary[field_key])
|
||||
else:
|
||||
for field_key, field_value in dictionary['fields'].items():
|
||||
point.field(field_key, field_value)
|
||||
|
||||
record_time_key = kwargs.get('record_time_key', 'time')
|
||||
if record_time_key in dictionary:
|
||||
point.time(dictionary[record_time_key], write_precision=write_precision)
|
||||
|
||||
_field_types = kwargs.get('field_types', {})
|
||||
if 'field_types' in dictionary:
|
||||
_field_types = dictionary['field_types']
|
||||
# Map API fields types to Line Protocol types postfix:
|
||||
# - int: 'i'
|
||||
# - uint: 'u'
|
||||
# - float: ''
|
||||
point._field_types = dict(map(
|
||||
lambda item: (item[0], 'i' if item[1] == 'int' else 'u' if item[1] == 'uint' else ''),
|
||||
_field_types.items()
|
||||
))
|
||||
|
||||
return point
|
||||
|
||||
def __init__(self, measurement_name):
|
||||
"""Initialize defaults."""
|
||||
self._tags = {}
|
||||
self._fields = {}
|
||||
self._name = measurement_name
|
||||
self._time = None
|
||||
self._write_precision = DEFAULT_WRITE_PRECISION
|
||||
self._field_types = {}
|
||||
|
||||
def time(self, time, write_precision=DEFAULT_WRITE_PRECISION):
|
||||
"""
|
||||
Specify timestamp for DataPoint with declared precision.
|
||||
|
||||
If time doesn't have specified timezone we assume that timezone is UTC.
|
||||
|
||||
Examples::
|
||||
Point.measurement("h2o").field("val", 1).time("2009-11-10T23:00:00.123456Z")
|
||||
Point.measurement("h2o").field("val", 1).time(1257894000123456000)
|
||||
Point.measurement("h2o").field("val", 1).time(datetime(2009, 11, 10, 23, 0, 0, 123456))
|
||||
Point.measurement("h2o").field("val", 1).time(1257894000123456000, write_precision=WritePrecision.NS)
|
||||
|
||||
|
||||
:param time: the timestamp for your data
|
||||
:param write_precision: sets the precision for the supplied time values
|
||||
:return: this point
|
||||
"""
|
||||
self._write_precision = write_precision
|
||||
self._time = time
|
||||
return self
|
||||
|
||||
def tag(self, key, value):
|
||||
"""Add tag with key and value."""
|
||||
self._tags[key] = value
|
||||
return self
|
||||
|
||||
def field(self, field, value):
|
||||
"""Add field with key and value."""
|
||||
self._fields[field] = value
|
||||
return self
|
||||
|
||||
def to_line_protocol(self, precision=None):
|
||||
"""
|
||||
Create LineProtocol.
|
||||
|
||||
:param precision: required precision of LineProtocol. If it's not set then use the precision from ``Point``.
|
||||
"""
|
||||
_measurement = _escape_key(self._name, _ESCAPE_MEASUREMENT)
|
||||
if _measurement.startswith("#"):
|
||||
message = f"""The measurement name '{_measurement}' start with '#'.
|
||||
|
||||
The output Line protocol will be interpret as a comment by InfluxDB. For more info see:
|
||||
- https://docs.influxdata.com/influxdb/latest/reference/syntax/line-protocol/#comments
|
||||
"""
|
||||
warnings.warn(message, SyntaxWarning)
|
||||
_tags = _append_tags(self._tags)
|
||||
_fields = _append_fields(self._fields, self._field_types)
|
||||
if not _fields:
|
||||
return ""
|
||||
_time = _append_time(self._time, self._write_precision if precision is None else precision)
|
||||
|
||||
return f"{_measurement}{_tags}{_fields}{_time}"
|
||||
|
||||
@property
|
||||
def write_precision(self):
|
||||
"""Get precision."""
|
||||
return self._write_precision
|
||||
|
||||
@classmethod
|
||||
def set_str_rep(cls, rep_function):
|
||||
"""Set the string representation for all Points."""
|
||||
cls.__str___rep = rep_function
|
||||
|
||||
def __str__(self):
|
||||
"""Create string representation of this Point."""
|
||||
return self.to_line_protocol()
|
||||
|
||||
def __eq__(self, other):
|
||||
"""Return true iff other is equal to self."""
|
||||
if not isinstance(other, Point):
|
||||
return False
|
||||
# assume points are equal iff their instance fields are equal
|
||||
return (self._tags == other._tags and
|
||||
self._fields == other._fields and
|
||||
self._name == other._name and
|
||||
self._time == other._time and
|
||||
self._write_precision == other._write_precision and
|
||||
self._field_types == other._field_types)
|
||||
|
||||
|
||||
def _append_tags(tags):
|
||||
_return = []
|
||||
for tag_key, tag_value in sorted(tags.items()):
|
||||
|
||||
if tag_value is None:
|
||||
continue
|
||||
|
||||
tag = _escape_key(tag_key)
|
||||
value = _escape_tag_value(tag_value)
|
||||
if tag != '' and value != '':
|
||||
_return.append(f'{tag}={value}')
|
||||
|
||||
return f"{',' if _return else ''}{','.join(_return)} "
|
||||
|
||||
|
||||
def _append_fields(fields, field_types):
|
||||
_return = []
|
||||
|
||||
for field, value in sorted(fields.items()):
|
||||
if value is None:
|
||||
continue
|
||||
|
||||
if isinstance(value, float) or isinstance(value, Decimal) or _np_is_subtype(value, 'float'):
|
||||
if not math.isfinite(value):
|
||||
continue
|
||||
s = str(value)
|
||||
# It's common to represent whole numbers as floats
|
||||
# and the trailing ".0" that Python produces is unnecessary
|
||||
# in line-protocol, inconsistent with other line-protocol encoders,
|
||||
# and takes more space than needed, so trim it off.
|
||||
if s.endswith('.0'):
|
||||
s = s[:-2]
|
||||
_return.append(f'{_escape_key(field)}={s}')
|
||||
elif (isinstance(value, int) or _np_is_subtype(value, 'int')) and not isinstance(value, bool):
|
||||
_type = field_types.get(field, "i")
|
||||
_return.append(f'{_escape_key(field)}={str(value)}{_type}')
|
||||
elif isinstance(value, bool):
|
||||
_return.append(f'{_escape_key(field)}={str(value).lower()}')
|
||||
elif isinstance(value, str):
|
||||
_return.append(f'{_escape_key(field)}="{_escape_string(value)}"')
|
||||
else:
|
||||
raise ValueError(f'Type: "{type(value)}" of field: "{field}" is not supported.')
|
||||
|
||||
return f"{','.join(_return)}"
|
||||
|
||||
|
||||
def _append_time(time, write_precision) -> str:
|
||||
if time is None:
|
||||
return ''
|
||||
return f" {int(_convert_timestamp(time, write_precision))}"
|
||||
|
||||
|
||||
def _escape_key(tag, escape_list=None) -> str:
|
||||
if escape_list is None:
|
||||
escape_list = _ESCAPE_KEY
|
||||
return str(tag).translate(escape_list)
|
||||
|
||||
|
||||
def _escape_tag_value(value) -> str:
|
||||
ret = _escape_key(value)
|
||||
if ret.endswith('\\'):
|
||||
ret += ' '
|
||||
return ret
|
||||
|
||||
|
||||
def _escape_string(value) -> str:
|
||||
return str(value).translate(_ESCAPE_STRING)
|
||||
|
||||
|
||||
def _convert_timestamp(timestamp, precision=DEFAULT_WRITE_PRECISION):
|
||||
date_helper = get_date_helper()
|
||||
if isinstance(timestamp, Integral):
|
||||
return timestamp # assume precision is correct if timestamp is int
|
||||
|
||||
if isinstance(timestamp, str):
|
||||
timestamp = date_helper.parse_date(timestamp)
|
||||
|
||||
if isinstance(timestamp, timedelta) or isinstance(timestamp, datetime):
|
||||
|
||||
if isinstance(timestamp, datetime):
|
||||
timestamp = date_helper.to_utc(timestamp) - EPOCH
|
||||
|
||||
ns = date_helper.to_nanoseconds(timestamp)
|
||||
|
||||
if precision is None or precision == WritePrecision.NS:
|
||||
return ns
|
||||
elif precision == WritePrecision.US:
|
||||
return ns / 1e3
|
||||
elif precision == WritePrecision.MS:
|
||||
return ns / 1e6
|
||||
elif precision == WritePrecision.S:
|
||||
return ns / 1e9
|
||||
|
||||
raise ValueError(timestamp)
|
||||
|
||||
|
||||
def _np_is_subtype(value, np_type):
|
||||
if not _HAS_NUMPY or not hasattr(value, 'dtype'):
|
||||
return False
|
||||
|
||||
if np_type == 'float':
|
||||
return np.issubdtype(value, np.floating)
|
||||
elif np_type == 'int':
|
||||
return np.issubdtype(value, np.integer)
|
||||
return False
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Implementation for Retry strategy during HTTP requests."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from itertools import takewhile
|
||||
from random import random
|
||||
from typing import Callable
|
||||
|
||||
from urllib3 import Retry
|
||||
from urllib3.exceptions import MaxRetryError, ResponseError
|
||||
|
||||
from influxdb_client.client.exceptions import InfluxDBError
|
||||
|
||||
logger = logging.getLogger('influxdb_client.client.write.retry')
|
||||
|
||||
|
||||
class WritesRetry(Retry):
|
||||
"""
|
||||
Writes retry configuration.
|
||||
|
||||
The next delay is computed as random value between range
|
||||
`retry_interval * exponential_base^(attempts-1)` and `retry_interval * exponential_base^(attempts)
|
||||
|
||||
Example:
|
||||
for retry_interval=5, exponential_base=2, max_retry_delay=125, total=5
|
||||
retry delays are random distributed values within the ranges of
|
||||
[5-10, 10-20, 20-40, 40-80, 80-125]
|
||||
"""
|
||||
|
||||
def __init__(self, jitter_interval=0, max_retry_delay=125, exponential_base=2, max_retry_time=180, total=5,
|
||||
retry_interval=5, retry_callback: Callable[[Exception], int] = None, **kw):
|
||||
"""
|
||||
Initialize defaults.
|
||||
|
||||
:param int jitter_interval: random milliseconds when retrying writes
|
||||
:param num max_retry_delay: maximum delay when retrying write in seconds
|
||||
:param int max_retry_time: maximum total retry timeout in seconds,
|
||||
attempt after this timout throws MaxRetryError
|
||||
:param int total: maximum number of retries
|
||||
:param num retry_interval: initial first retry delay range in seconds
|
||||
:param int exponential_base: base for the exponential retry delay,
|
||||
:param Callable[[Exception], int] retry_callback: the callable ``callback`` to run after retryable
|
||||
error occurred.
|
||||
The callable must accept one argument:
|
||||
- `Exception`: an retryable error
|
||||
"""
|
||||
super().__init__(**kw)
|
||||
self.jitter_interval = jitter_interval
|
||||
self.total = total
|
||||
self.retry_interval = retry_interval
|
||||
self.max_retry_delay = max_retry_delay
|
||||
self.max_retry_time = max_retry_time
|
||||
self.exponential_base = exponential_base
|
||||
self.retry_timeout = datetime.now() + timedelta(seconds=max_retry_time)
|
||||
self.retry_callback = retry_callback
|
||||
|
||||
def new(self, **kw):
|
||||
"""Initialize defaults."""
|
||||
if 'jitter_interval' not in kw:
|
||||
kw['jitter_interval'] = self.jitter_interval
|
||||
if 'retry_interval' not in kw:
|
||||
kw['retry_interval'] = self.retry_interval
|
||||
if 'max_retry_delay' not in kw:
|
||||
kw['max_retry_delay'] = self.max_retry_delay
|
||||
if 'max_retry_time' not in kw:
|
||||
kw['max_retry_time'] = self.max_retry_time
|
||||
if 'exponential_base' not in kw:
|
||||
kw['exponential_base'] = self.exponential_base
|
||||
if 'retry_callback' not in kw:
|
||||
kw['retry_callback'] = self.retry_callback
|
||||
|
||||
new = super().new(**kw)
|
||||
new.retry_timeout = self.retry_timeout
|
||||
return new
|
||||
|
||||
def is_retry(self, method, status_code, has_retry_after=False):
|
||||
"""is_retry doesn't require retry_after header. If there is not Retry-After we will use backoff."""
|
||||
if not self._is_method_retryable(method):
|
||||
return False
|
||||
|
||||
return self.total and (status_code >= 429)
|
||||
|
||||
def get_backoff_time(self):
|
||||
"""Variant of exponential backoff with initial and max delay and a random jitter delay."""
|
||||
# We want to consider only the last consecutive errors sequence (Ignore redirects).
|
||||
consecutive_errors_len = len(
|
||||
list(
|
||||
takewhile(lambda x: x.redirect_location is None, reversed(self.history))
|
||||
)
|
||||
)
|
||||
# First fail doesn't increase backoff
|
||||
consecutive_errors_len -= 1
|
||||
if consecutive_errors_len < 0:
|
||||
return 0
|
||||
|
||||
range_start = self.retry_interval
|
||||
range_stop = self.retry_interval * self.exponential_base
|
||||
|
||||
i = 1
|
||||
while i <= consecutive_errors_len:
|
||||
i += 1
|
||||
range_start = range_stop
|
||||
range_stop = range_stop * self.exponential_base
|
||||
if range_stop > self.max_retry_delay:
|
||||
break
|
||||
|
||||
if range_stop > self.max_retry_delay:
|
||||
range_stop = self.max_retry_delay
|
||||
|
||||
return range_start + (range_stop - range_start) * self._random()
|
||||
|
||||
def get_retry_after(self, response):
|
||||
"""Get the value of Retry-After header and append random jitter delay."""
|
||||
retry_after = super().get_retry_after(response)
|
||||
if retry_after:
|
||||
retry_after += self._jitter_delay()
|
||||
return retry_after
|
||||
|
||||
def increment(self, method=None, url=None, response=None, error=None, _pool=None, _stacktrace=None):
|
||||
"""Return a new Retry object with incremented retry counters."""
|
||||
if self.retry_timeout < datetime.now():
|
||||
raise MaxRetryError(_pool, url, error or ResponseError("max_retry_time exceeded"))
|
||||
|
||||
new_retry = super().increment(method, url, response, error, _pool, _stacktrace)
|
||||
|
||||
if response is not None:
|
||||
parsed_error = InfluxDBError(response=response)
|
||||
elif error is not None:
|
||||
parsed_error = error
|
||||
else:
|
||||
parsed_error = f"Failed request to: {url}"
|
||||
|
||||
message = f"The retriable error occurred during request. Reason: '{parsed_error}'."
|
||||
if isinstance(parsed_error, InfluxDBError):
|
||||
message += f" Retry in {parsed_error.retry_after}s."
|
||||
|
||||
if self.retry_callback:
|
||||
self.retry_callback(parsed_error)
|
||||
|
||||
logger.warning(message)
|
||||
|
||||
return new_retry
|
||||
|
||||
def _jitter_delay(self):
|
||||
return self.jitter_interval * random()
|
||||
|
||||
def _random(self):
|
||||
return random()
|
||||
Reference in New Issue
Block a user