From 2088d2879bf00c4b419e0294adea1f3a86d3866c Mon Sep 17 00:00:00 2001 From: aleksandrashlychkova Date: Sat, 18 May 2019 22:43:53 +0300 Subject: [PATCH 1/5] test system with docker executor --- .circleci/config.yml | 85 +++ .idea/inclickdb.iml | 11 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + .idea/workspace.xml | 815 +++++++++++++++++++++++++++ Generator/Dockerfile | 6 + Generator/generator.sh | 10 + Image/Dockerfile | 6 + Image/image.sh | 6 + demo2/.gitignore | 1 + demo2/Dockerfile | 5 + demo2/README.md | 18 + demo2/docker-compose.yaml | 26 + demo2/reciever_python/Dockerfile | 5 + demo2/reciever_python/Parser.py | 89 +++ demo2/reciever_python/add_metrics.sh | 5 + demo2/reciever_python/init.py | 31 + demo2/reciever_python/insert.py | 31 + demo2/reciever_python/tmp.txt | 1 + 19 files changed, 1165 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 .idea/inclickdb.iml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml create mode 100644 Generator/Dockerfile create mode 100644 Generator/generator.sh create mode 100644 Image/Dockerfile create mode 100644 Image/image.sh create mode 100644 demo2/.gitignore create mode 100644 demo2/Dockerfile create mode 100644 demo2/README.md create mode 100644 demo2/docker-compose.yaml create mode 100644 demo2/reciever_python/Dockerfile create mode 100644 demo2/reciever_python/Parser.py create mode 100644 demo2/reciever_python/add_metrics.sh create mode 100644 demo2/reciever_python/init.py create mode 100644 demo2/reciever_python/insert.py create mode 100644 demo2/reciever_python/tmp.txt diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..c039552 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,85 @@ +version: 2 +jobs: + + build_demo_0: + + docker: + - image: circleci/python:3.7.0 + + steps: + + - checkout + - setup_remote_docker + + + - run: + name: compose + command: | + cd stand + docker-compose up + background: true + + - run: + name: generator + command: | + docker build -t generator ./Generator + docker run --network container:stand_clickhouse_1 generator + + background: true + + - run: + name: image + command: | + docker build -t image ./Image + docker run --network container:stand_grafana_1: image + + - run: + name: copy image + command: | + docker cp image:/snapshots/output.png stand/snapshots/output.png + + + - store_artifacts: + path: stand/snapshots + + + + + build_demo_2: + + docker: + - image: circleci/python:3.7.0 + + steps: + + - checkout + - setup_remote_docker + + - run: + name: compose + command: | + cd demo2 + ls + docker-compose up + background: true + + + - run: + name: generator + command: | + docker build -t generator ./Generator + docker ps + docker run --network container:demo2_data-receiver_1 generator + + +workflows: + version: 2 + demo0: + jobs: + - build_demo_0 +# demo1: +# jobs: +# - build_demo_1 + demo2: + jobs: + - build_demo_2 diff --git a/.idea/inclickdb.iml b/.idea/inclickdb.iml new file mode 100644 index 0000000..6711606 --- /dev/null +++ b/.idea/inclickdb.iml @@ -0,0 +1,11 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..8cca42d --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..ae1abc2 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,815 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1558118172383 - - - 1558189560571 - - - 1558189667435 - - - 1558189699145 - - - 1558189850662 - - - 1558190257688 - - - 1558190307601 - - - 1558190398272 - - - 1558190451196 - - - 1558190510023 - - - 1558190902831 - - - 1558191020979 - - - 1558191401375 - - - 1558191632687 - - - 1558192322810 - - - 1558193116416 - - - 1558193346275 - - - 1558193497176 - - - 1558193669753 - - - 1558193767114 - - - 1558193895411 - - - 1558194112264 - - - 1558195070847 - - - 1558195139292 - - - 1558195223685 - - - 1558195568597 - - - 1558196317866 - - - 1558196931787 - - - 1558197072949 - - - 1558197208051 - - - 1558198172552 - - - 1558198255029 - - - 1558198297375 - - - 1558198342830 - - - 1558198400462 - - - 1558198437291 - - - 1558198607741 - - - 1558198849766 - - - 1558199536096 - - - 1558199868148 - - - 1558199982357 - - - 1558200398483 - - - 1558200817031 - - - 1558200989982 - - - 1558201739241 - - - 1558201889316 - - - 1558202231740 - - - 1558202633590 - - - 1558203698891 - - - 1558206339579 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Generator/Dockerfile b/Generator/Dockerfile index 74a1ae3..a9b62b9 100644 --- a/Generator/Dockerfile +++ b/Generator/Dockerfile @@ -1,6 +1,5 @@ FROM circleci/python:3.7.0 ADD . / -RUN sleep 60 -CMD bash generator.sh \ No newline at end of file +CMD bash generator.sh diff --git a/Image/image.sh b/Image/image.sh deleted file mode 100644 index 7cbc4ea..0000000 --- a/Image/image.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - - -key=$(cat grafana-key/key.txt) -curl 'http://stand_grafana_1:3000/render/d-solo/JE7rYbjmz/dh?orgId=1&panelId=2&width=1000&height=500&tz=Europe%2FMoscow' -H 'Authorization: Bearer '$key --compressed > ./snapshots/output.png -sleep 20 diff --git a/Image/Dockerfile b/Screenshot/Dockerfile similarity index 56% rename from Image/Dockerfile rename to Screenshot/Dockerfile index aea8f7c..27e3c72 100644 --- a/Image/Dockerfile +++ b/Screenshot/Dockerfile @@ -1,6 +1,5 @@ FROM circleci/python:3.7.0 ADD . / -RUN sleep 60 -CMD sh image.sh \ No newline at end of file +CMD sh image.sh diff --git a/Screenshot/image.sh b/Screenshot/image.sh new file mode 100644 index 0000000..c6d7222 --- /dev/null +++ b/Screenshot/image.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +key=$(cat grafana-key/key.txt) +curl 'http://localhost:3000/render/\ +d-solo/JE7rYbjmz/dh?orgId=\ +1&panelId=\ +2&width=\ +1000&height=500' -H 'Authorization: Bearer '$key --compressed > ./snapshots/output.png diff --git a/demo2/Dockerfile b/demo2/Dockerfile index 41366ad..a36648f 100644 --- a/demo2/Dockerfile +++ b/demo2/Dockerfile @@ -2,4 +2,4 @@ FROM circleci/python:3.7.0 ADD . / -CMD sh generator.sh \ No newline at end of file +CMD sh generator.sh diff --git a/demo2/README.md b/demo2/README.md index 34da3a1..0f01793 100644 --- a/demo2/README.md +++ b/demo2/README.md @@ -1,7 +1,7 @@ ### Store metrics #### What is it? -This stand does the following: +This demo does the following: - Generates some graphite data - parses it - Sends it to [clickhouse] @@ -10,9 +10,4 @@ This stand does the following: Do the following steps: - `docker-compose up` -- `bash generator.sh`, `bash generator2.sh` (for generating data) - -#### TODO - -- Add .sh with cron -- Add grafana \ No newline at end of file +- `bash generator.sh`(for generating data) diff --git a/demo2/docker-compose.yaml b/demo2/docker-compose.yaml index 6df8259..8378424 100644 --- a/demo2/docker-compose.yaml +++ b/demo2/docker-compose.yaml @@ -16,11 +16,8 @@ data-receiver: - "2003:2003/udp" # plain udp - "2004:2004" # pickle - "2006:2006" # prometheus remote write -<<<<<<< HEAD + entrypoint: sh add_metrics.sh -======= - entrypoint: sh /reciever_python/add_metrics.sh ->>>>>>> ab66b9d... - + links: - clickhouse - diff --git a/demo2/reciever_python/Parser.py b/demo2/reciever_python/Parser.py index efe2614..3d93fa5 100644 --- a/demo2/reciever_python/Parser.py +++ b/demo2/reciever_python/Parser.py @@ -1,19 +1,22 @@ +""" +parsing data with templates +parsing data with templates +""" import datetime -def parse_tagged_data(data, Taglist=[]): +def parse_tagged_data(data, taglist=[]): - ''' + """ :param data: string form of mectic :param Taglist: list of column in table :return: insert_data - dict of data to insert into table; - Tag_to_add - tags that are not in Taglist; + tag_to_add - tags that are not in taglist; tags - tags for insert query - to_do: add type check - ''' + """ - if type(data) != str: + if isinstance(data) != str: data = data.decode("utf-8") data = data.split() @@ -21,69 +24,86 @@ def parse_tagged_data(data, Taglist=[]): return None, None, None insert_data = dict() - insert_data['timestmp'] = int(data[2]) - insert_data['last_volume'] = int(data[1]) - data = data[0].split(';') - insert_data['path'] = data[0] - Tag_to_add = [] + insert_data["timestmp"] = int(data[2]) + insert_data["last_volume"] = int(data[1]) + data = data[0].split(";") + insert_data["path"] = data[0] + tag_to_add = [] data = data[1:] - tags = 'timestmp, path, last_volume' + tags = "timestmp, path, last_volume" for tag_value in data: - tag = tag_value.split('=')[0] - value = int(tag_value.split('=')[1]) + tag = tag_value.split("=")[0] + value = int(tag_value.split("=")[1]) - tags += (', ' + tag) + tags += ", " + tag insert_data[tag] = value - if tag not in Taglist: - Tag_to_add.append(tag) + if tag not in taglist: + tag_to_add.append(tag) - return insert_data, Tag_to_add, tags + return insert_data, tag_to_add, tags -def is_match(tmp, data, names, Taglist=[]): - tmp = tmp.split('.') +def is_match(tmp, data, names, taglist=[]): + """ + :param tmp: string template + :param taglist: list of column in table + :return: + insert_data - dict of data to insert into table; + tag_to_add - tags that are not in taglist; + tags - tags for insert query + """ + tmp = tmp.split(".") data = data.split() - data_ = data[0].split('.') - names = names.split('.') + data_ = data[0].split(".") + names = names.split(".") insert_data = dict() - path = '' - last_volume = '' - Tag_to_add = [] - tags = 'timestmp, path, last_volume' + path = "" + last_volume = "" + tag_to_add = [] + tags = "timestmp, path, last_volume" for i in range(len(tmp)): - if tmp[i] != '*': + if tmp[i] != "*": if data_[i] != tmp[i]: return None, None, None - if names[i] == 'measurement': - path += data_[i] + '.' - elif names[i] == 'field': + if names[i] == "measurement": + path += data_[i] + "." + elif names[i] == "field": last_volume += data_[i] - elif names[i] != '': + elif names[i] != "": insert_data[names[i]] = data_[i] - if names[i] not in Taglist: - Tag_to_add.append(names[i]) - tags += (', ' + names[i]) + if names[i] not in taglist: + tag_to_add.append(names[i]) + tags += ", " + names[i] insert_data[last_volume] = data[-1] - insert_data['path'] = path - insert_data['timestmp'] = int(str(datetime.datetime.timestamp(datetime.datetime.now())).split('.')[0]) - return insert_data, Tag_to_add, tags - - -def parse_tamplate(filename, data, Taglist=[]): - if type(filename) == str: - f = open(filename, 'r') + insert_data["path"] = path + tmsp = str(datetime.datetime.timestamp(datetime.datetime.now())).split(".") + insert_data["timestmp"] = int(tmsp[0]) + return insert_data, tag_to_add, tags + + +def parse_tamplate(filename, data, taglist=[]): + """ + :param filename: string name of file + :param taglist: list of column in table + :param data: data to parse + :return: + insert_data - dict of data to insert into table; + tag_to_add - tags that are not in taglist; + tags - tags for insert query + """ + if isinstance(filename) == str: + file = open(filename, "r") else: - f = filename + file = filename - for line in f: - L = line.split() - insert_data, Tag_to_add, tags = is_match(L[0], data, L[1], Taglist) + for line in file: + linesp = line.split() + insert_data, tag_to_add, tags = is_match(linesp[0], data, linesp[1], taglist) if insert_data: - return insert_data, Tag_to_add, tags + return insert_data, tag_to_add, tags return None, None, None - diff --git a/demo2/reciever_python/add_metrics.sh b/demo2/reciever_python/add_metrics.sh index ba31442..f19c488 100644 --- a/demo2/reciever_python/add_metrics.sh +++ b/demo2/reciever_python/add_metrics.sh @@ -1,5 +1,3 @@ #!/usr/bin/env bash - -ls python init.py -python insert.py \ No newline at end of file +python insert.py diff --git a/demo2/reciever_python/init.py b/demo2/reciever_python/init.py index ff20552..654fb7d 100644 --- a/demo2/reciever_python/init.py +++ b/demo2/reciever_python/init.py @@ -1,8 +1,7 @@ -import socket from clickhouse_driver.client import Client -def init(table_name='events'): +def init(table_name="events"): """ :param table_name:name of table for mecrics :return: ok @@ -11,20 +10,21 @@ def init(table_name='events'): """ # connection to clickhous - sock = socket.socket() - PORT = 2003 - sock.bind(("", PORT)) - client = Client('clickhouse') + client = Client("clickhouse") # creating database and table print(client.execute("CREATE DATABASE IF NOT EXISTS " + table_name)) - print(client.execute( - 'CREATE TABLE IF NOT EXISTS events.tmp(\n \ + print( + client.execute( + "CREATE TABLE IF NOT EXISTS events.tmp(\n \ timestmp UInt32, \n \ last_volume UInt32, \n \ path String \n \ - ) ENGINE MergeTree() PARTITION BY timestmp ORDER BY timestmp SETTINGS index_granularity=8192;')) - return 'created: events.tmp' + ) ENGINE MergeTree() PARTITION BY\ + timestmp ORDER BY timestmp SETTINGS index_granularity=8192;" + ) + ) + return "created: events.tmp" if __name__ == "__main__": diff --git a/demo2/reciever_python/insert.py b/demo2/reciever_python/insert.py index efaddee..85a8830 100644 --- a/demo2/reciever_python/insert.py +++ b/demo2/reciever_python/insert.py @@ -2,30 +2,43 @@ from clickhouse_driver.client import Client from Parser import parse_tagged_data -sock = socket.socket() -PORT = 2003 -sock.bind(("", PORT)) -Taglist = [] if __name__ == "__main__": + SOCK = socket.socket() + PORT = 2003 + SOCK.bind(("", PORT)) + + TAGLIST = [] + # connection to clickhouse - client = Client('clickhouse') + CLIENT = Client("clickhouse") for i in range(100): - sock.listen(1) - conn, addr = sock.accept() + SOCK.listen(1) + conn, addr = SOCK.accept() data = conn.recv(512) insert_data, Tag_to_add, tags = parse_tagged_data(data) if Tag_to_add != [] and Tag_to_add: for tag in Tag_to_add: - Taglist.append(tag) - (client.execute('ALTER TABLE events.tmp ADD COLUMN IF NOT EXISTS ' + tag + ' UInt32 AFTER path')) + TAGLIST.append(tag) + ( + CLIENT.execute( + "ALTER TABLE events.tmp ADD \ + COLUMN IF NOT EXISTS " + + tag + + " UInt32 AFTER path" + ) + ) if insert_data: - (client.execute('INSERT INTO events.tmp (' + tags + ') VALUES', [insert_data])) + ( + CLIENT.execute( + "INSERT INTO events.tmp (" + tags + ") VALUES", [insert_data] + ) + ) - print(client.execute('SELECT * FROM events.tmp')) + print(CLIENT.execute("SELECT * FROM events.tmp")) diff --git a/demo2/reciever_python/tmp.txt b/demo2/reciever_python/tmp.txt index d111a55..113997a 100644 --- a/demo2/reciever_python/tmp.txt +++ b/demo2/reciever_python/tmp.txt @@ -1 +1 @@ -*.cgroups.*.cpu.* host.measurement.cgroup.measurement.field \ No newline at end of file +*.cgroups.*.cpu.* host.measurement.cgroup.measurement.field From 06387fedde11d3916cb7da25990679743e89ba36 Mon Sep 17 00:00:00 2001 From: aleksandrashlychkova Date: Sun, 19 May 2019 21:33:44 +0300 Subject: [PATCH 3/5] fixed --- .circleci/config.yml | 60 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c039552..038e3ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,6 @@ jobs: - checkout - setup_remote_docker - - run: name: compose command: | @@ -24,25 +23,60 @@ jobs: command: | docker build -t generator ./Generator docker run --network container:stand_clickhouse_1 generator - background: true - run: - name: image + name: screenshot command: | - docker build -t image ./Image - docker run --network container:stand_grafana_1: image + docker build -t screenshot ./Screenshot + docker run --network container:stand_grafana_1: screenshot - run: - name: copy image + name: copy screenshot command: | - docker cp image:/snapshots/output.png stand/snapshots/output.png - + docker cp screenshot:/snapshots/output.png stand/snapshots/output.png - store_artifacts: path: stand/snapshots + build_demo_1: + + docker: + - image: circleci/python:3.7.0 + + steps: + + - checkout + - setup_remote_docker + + - run: + name: compose + command: | + cd demo + docker-compose up + background: true + + - run: + name: generator + command: | + docker build -t generator ./Generator + docker run --network container:stand_clickhouse_1 generator + background: true + + - run: + name: screenshot + command: | + docker build -t screenshot ./Screenshot + docker run --network container:stand_grafana_1: screenshot + + - run: + name: copy screenshot + command: | + docker cp screenshot:/snapshots/output.png stand/snapshots/output.png + + - store_artifacts: + path: stand/snapshots build_demo_2: @@ -63,7 +97,6 @@ jobs: docker-compose up background: true - - run: name: generator command: | @@ -74,12 +107,15 @@ jobs: workflows: version: 2 + demo0: jobs: - build_demo_0 -# demo1: -# jobs: -# - build_demo_1 + + demo1: + jobs: + - build_demo_1 + demo2: jobs: - build_demo_2 From 4915f20cc3d4bd793dfc3052b3fa88023cfceccf Mon Sep 17 00:00:00 2001 From: aleksandrashlychkova Date: Sun, 19 May 2019 22:55:47 +0300 Subject: [PATCH 4/5] test system with docker executor (iteration 2) --- .circleci/config.yml | 48 ++----------------- .gitignore | 1 + Screenshot/image.sh | 11 ++--- demo2/.gitignore | 1 - demo2/reciever_python/add_metrics.sh | 1 + demo2/reciever_python/init.py | 1 + demo2/reciever_python/insert.py | 41 +++++++++------- .../{Parser.py => parser_for_taged.py} | 18 +++---- demo2/reciever_python/tmp.txt | 1 + 9 files changed, 47 insertions(+), 76 deletions(-) delete mode 100644 demo2/.gitignore rename demo2/reciever_python/{Parser.py => parser_for_taged.py} (88%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 038e3ed..87fcfe9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,45 +1,6 @@ version: 2 jobs: - build_demo_0: - - docker: - - image: circleci/python:3.7.0 - - steps: - - - checkout - - setup_remote_docker - - - run: - name: compose - command: | - cd stand - docker-compose up - background: true - - - run: - name: generator - command: | - docker build -t generator ./Generator - docker run --network container:stand_clickhouse_1 generator - background: true - - - run: - name: screenshot - command: | - docker build -t screenshot ./Screenshot - docker run --network container:stand_grafana_1: screenshot - - - run: - name: copy screenshot - command: | - docker cp screenshot:/snapshots/output.png stand/snapshots/output.png - - - store_artifacts: - path: stand/snapshots - - build_demo_1: docker: @@ -61,22 +22,22 @@ jobs: name: generator command: | docker build -t generator ./Generator - docker run --network container:stand_clickhouse_1 generator + docker run --network container:demo_clickhouse_1 generator background: true - run: name: screenshot command: | docker build -t screenshot ./Screenshot - docker run --network container:stand_grafana_1: screenshot + docker run --network container:demo_grafana_1: screenshot - run: name: copy screenshot command: | - docker cp screenshot:/snapshots/output.png stand/snapshots/output.png + docker cp screenshot:/snapshots/output.png demo/snapshots/output.png - store_artifacts: - path: stand/snapshots + path: demo/snapshots build_demo_2: @@ -104,7 +65,6 @@ jobs: docker ps docker run --network container:demo2_data-receiver_1 generator - workflows: version: 2 diff --git a/.gitignore b/.gitignore index 5ac9272..4ea79a8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ stand/data stand/snapshots stand/grafana-key/key.txt +.idea diff --git a/Screenshot/image.sh b/Screenshot/image.sh index c6d7222..a8832d4 100644 --- a/Screenshot/image.sh +++ b/Screenshot/image.sh @@ -1,8 +1,7 @@ #!/bin/sh -key=$(cat grafana-key/key.txt) -curl 'http://localhost:3000/render/\ -d-solo/JE7rYbjmz/dh?orgId=\ -1&panelId=\ -2&width=\ -1000&height=500' -H 'Authorization: Bearer '$key --compressed > ./snapshots/output.png +key=$(<../demo2/grafana-key/key.txt) +req='/d-solo/JE7rYbjmz/dh?orgId=1&panelId=2&width=1000&height=500' +curl 'http://localhost:3000/render'$req \ +-H 'Authorization: Bearer '$key \ +--compressed > ./snapshots/output.png diff --git a/demo2/.gitignore b/demo2/.gitignore deleted file mode 100644 index 485dee6..0000000 --- a/demo2/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.idea diff --git a/demo2/reciever_python/add_metrics.sh b/demo2/reciever_python/add_metrics.sh index f19c488..99044a2 100644 --- a/demo2/reciever_python/add_metrics.sh +++ b/demo2/reciever_python/add_metrics.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash + python init.py python insert.py diff --git a/demo2/reciever_python/init.py b/demo2/reciever_python/init.py index 654fb7d..8d795c3 100644 --- a/demo2/reciever_python/init.py +++ b/demo2/reciever_python/init.py @@ -1,3 +1,4 @@ +"""init table""" from clickhouse_driver.client import Client diff --git a/demo2/reciever_python/insert.py b/demo2/reciever_python/insert.py index 85a8830..933ca24 100644 --- a/demo2/reciever_python/insert.py +++ b/demo2/reciever_python/insert.py @@ -1,32 +1,37 @@ +"""insert tagged data into table""" import socket from clickhouse_driver.client import Client -from Parser import parse_tagged_data +from parser_for_taged import parse_tagged_data -if __name__ == "__main__": +def main(): + """ + connects to clickhouse and calls parser and inserts data + :return: nothing + """ - SOCK = socket.socket() - PORT = 2003 - SOCK.bind(("", PORT)) + sock = socket.socket() + port = 2003 + sock.bind(("", port)) - TAGLIST = [] + taglist = [] # connection to clickhouse - CLIENT = Client("clickhouse") + client = Client("clickhouse") for i in range(100): - SOCK.listen(1) - conn, addr = SOCK.accept() + sock.listen(1) + conn, addr = sock.accept() data = conn.recv(512) - insert_data, Tag_to_add, tags = parse_tagged_data(data) + insert_data, tag_to_add, tags, taglist = parse_tagged_data(data, taglist) - if Tag_to_add != [] and Tag_to_add: - for tag in Tag_to_add: - TAGLIST.append(tag) + if tag_to_add != [] and tag_to_add: + for tag in tag_to_add: + taglist.append(tag) ( - CLIENT.execute( + client.execute( "ALTER TABLE events.tmp ADD \ COLUMN IF NOT EXISTS " + tag @@ -36,9 +41,13 @@ if insert_data: ( - CLIENT.execute( + client.execute( "INSERT INTO events.tmp (" + tags + ") VALUES", [insert_data] ) ) - print(CLIENT.execute("SELECT * FROM events.tmp")) + print(client.execute("SELECT * FROM events.tmp")) + + +if __name__ == "__main__": + main() diff --git a/demo2/reciever_python/Parser.py b/demo2/reciever_python/parser_for_taged.py similarity index 88% rename from demo2/reciever_python/Parser.py rename to demo2/reciever_python/parser_for_taged.py index 3d93fa5..d21cfaf 100644 --- a/demo2/reciever_python/Parser.py +++ b/demo2/reciever_python/parser_for_taged.py @@ -5,7 +5,7 @@ import datetime -def parse_tagged_data(data, taglist=[]): +def parse_tagged_data(data, taglist): """ :param data: string form of mectic @@ -39,13 +39,14 @@ def parse_tagged_data(data, taglist=[]): tags += ", " + tag insert_data[tag] = value + new_tag_list = taglist[:] if tag not in taglist: - tag_to_add.append(tag) + new_tag_list.append(tag) - return insert_data, tag_to_add, tags + return insert_data, tag_to_add, tags, new_tag_list -def is_match(tmp, data, names, taglist=[]): +def is_match(tmp, data, names, taglist): """ :param tmp: string template :param taglist: list of column in table @@ -65,9 +66,8 @@ def is_match(tmp, data, names, taglist=[]): tags = "timestmp, path, last_volume" for i in range(len(tmp)): - if tmp[i] != "*": - if data_[i] != tmp[i]: - return None, None, None + if tmp[i] != "*" and data_[i] != tmp[i]: + return None, None, None if names[i] == "measurement": path += data_[i] + "." elif names[i] == "field": @@ -85,7 +85,7 @@ def is_match(tmp, data, names, taglist=[]): return insert_data, tag_to_add, tags -def parse_tamplate(filename, data, taglist=[]): +def parse_tamplate(filename, data, taglist): """ :param filename: string name of file :param taglist: list of column in table @@ -106,4 +106,4 @@ def parse_tamplate(filename, data, taglist=[]): if insert_data: return insert_data, tag_to_add, tags - return None, None, None + return None diff --git a/demo2/reciever_python/tmp.txt b/demo2/reciever_python/tmp.txt index 113997a..e990c8f 100644 --- a/demo2/reciever_python/tmp.txt +++ b/demo2/reciever_python/tmp.txt @@ -1 +1,2 @@ *.cgroups.*.cpu.* host.measurement.cgroup.measurement.field +*.*.path.* \ No newline at end of file From 0ade035a209d49f573dfb30a28147c716e3d8859 Mon Sep 17 00:00:00 2001 From: aleksandrashlychkova Date: Tue, 21 May 2019 14:49:36 +0300 Subject: [PATCH 5/5] isinstance fixed --- demo2/reciever_python/parser_for_taged.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demo2/reciever_python/parser_for_taged.py b/demo2/reciever_python/parser_for_taged.py index d21cfaf..e2ed1ab 100644 --- a/demo2/reciever_python/parser_for_taged.py +++ b/demo2/reciever_python/parser_for_taged.py @@ -16,7 +16,7 @@ def parse_tagged_data(data, taglist): tags - tags for insert query """ - if isinstance(data) != str: + if isinstance(data, str): data = data.decode("utf-8") data = data.split() @@ -95,7 +95,7 @@ def parse_tamplate(filename, data, taglist): tag_to_add - tags that are not in taglist; tags - tags for insert query """ - if isinstance(filename) == str: + if isinstance(filename, str): file = open(filename, "r") else: file = filename