Upgrade to 1.5.2, tooling for VPN API requests

This commit is contained in:
Quentin WEPHRE
2024-07-19 08:59:28 +02:00
parent 5819418f7c
commit 1a383c5302
25 changed files with 2026 additions and 1835 deletions

2
.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
I-Sight_Generated_Files*
DATAMODEL_*

View File

@@ -35,7 +35,7 @@ allowed_name_characters.append('.')
logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', filename='data_config_debug.log', filemode='w', level=logging.DEBUG, datefmt='%Y%m%d%H%M%S')
dir_name = 'I-Sight_Generated_Files'
input_datamodel = 'DATAMODEL_1.0.6_MCO2.xlsx'
input_datamodel = 'DATAMODEL_1.0.6_SASK.xlsx'
shell_script_name = dir_name + '/I-Sight_Configuration_'
if (os.path.isdir(dir_name)):
@@ -189,21 +189,15 @@ def jq_filter(current_device, dsh):
if filter in jq_filter_set and filter != "bitfield":
logging.debug("Creating standard telemetry topic " + filter + ".")
jq_data = {}
jq_data["enable"] = True
jq_data["enable"] = False
jq_data["properties"] = [{"key": "deviceType", "value": "AC_GATEWAY"}, {"key": "cdid", "value": current_device}]
jq_data["outputTopic"] = filter
if row['message_polling_interval']:
jq_data["pollingInterval"] = int(row['message_polling_interval'])
else:
jq_data["pollingInterval"] = int(1)
if row['message_time_limit'] and row['message_size_limit']:
jq_data["sendOutThreshold"] = {"size": int(row['message_size_limit']), "time": int(row['message_time_limit'])}
elif not row['message_time_limit'] and row['message_size_limit']:
jq_data["sendOutThreshold"] = {"size": int(row['message_size_limit']), "time": int(30)}
elif row['message_time_limit'] and not row['message_size_limit']:
jq_data["sendOutThreshold"] = {"size": int(256000), "time": int(row['message_time_limit'])}
else:
jq_data["sendOutThreshold"] = {"size": int(256000), "time": int(30)}
jq_data["sendOutThreshold"] = {"mode": "bySize", "size": int(128000), "time": int(30), "sizeIdleTimer": {"enable": True, "time": int(30)}}
jq_data["minPublishInterval"] = int(0)
jq_data["samplingMode"] = "allValues"
jq_data["customSamplingRate"] = False
jq_data["pollingInterval"] = int(0)
jq_data["onChange"] = False
final_filter = row["jq_filter"].replace("***device_name***", current_device)
cmd_list = {}
for metric in all_metrics:
@@ -245,7 +239,7 @@ def bitfield_jq_filter(current_device):
logging.debug("Creating bitfield specific telemetry topic" + metric['metric_name'] + " for " + str(assets_name[metric["remoteDevId"]]) + ".")
final_filter = {}
json_request = {}
json_request["enable"] = True
json_request["enable"] = False
json_request["properties"] = [{"key": "deviceType", "value": "AC_GATEWAY"}, {"key": "cdid", "value": current_device}]
json_request["outputTopic"] = metric["metric_name"]
json_request["pollingInterval"] = 60
@@ -336,7 +330,7 @@ def tpfunc_gen():
package = {}
package["name"] = "demoDataFunc"
package["enabled"] = True
package["enabled"] = False
package["trigger"] = {"driven":"timeDriven","timeDriven":{"mode":"boot","intervalSec":2,"cronJob":""}}
package["expose"] = {"tags": expose_tags}
package["executable"] = {"language":"python"}

View File

@@ -12,14 +12,14 @@ payload = '{"method":"GET", "path":"/device/general"}'
# pip install azure-iot-hub
# Authenticate to your Azure account
CONNECTION_STRING = "HostName=IotHub-CUBE-PROD.azure-devices.net;SharedAccessKeyName=iothubowner;SharedAccessKey= ..."
CONNECTION_STRING = "HostName=IotHub-CUBE-PROD.azure-devices.net;SharedAccessKeyName=iothubowner;SharedAccessKey=...="
if CONNECTION_STRING == "":
print("Provide a connection string for the Iot Hub before running the script!")
exit(13)
registry_manager = IoTHubRegistryManager.from_connection_string(CONNECTION_STRING)
query_spec = QuerySpecification(query="SELECT * FROM devices WHERE tags.site != 'QWE' AND tags.number != '0' AND capabilities.iotEdge = true")
query_spec = QuerySpecification(query="SELECT * FROM devices WHERE tags.site = 'MYRTLE' AND tags.number != '0' AND capabilities.iotEdge = true")
query_result = registry_manager.query_iot_hub(query_spec)
devices = []
@@ -38,7 +38,11 @@ for i in ordered_devices:
try:
direct_method = CloudToDeviceMethod(method_name=method_name, payload=json.loads(payload))
response = registry_manager.invoke_device_module_method(device_id=i[1], module_id=module_id, direct_method_request=direct_method)
print(str(i[2]), str(i[0]), str(i[1]), response.payload['data']['description'],sep=";")
#print(response.payload)
if str(i[1]) == str(response.payload['data']['hostName']):
print(str(i[2]), str(i[0]), str(i[1]), response.payload['data']['description'], response.payload['data']['firmwareVersion'], sep=";")
else:
print(str(i[2]), str(i[0]), str(i[1]), response.payload['data']['description'], response.payload['data']['hostName'], response.payload['data']['firmwareVersion'], sep=";")
except:
print(str(i[2]), str(i[0]), str(i[1]), "UNREACHABLE",sep=";")
else:

View File

@@ -11,7 +11,7 @@ def authenticate(device_ip, payload):
if response.status_code == 200:
token = response.json()["data"]["token"]
#print(f"Authentication successful. Token received: {token}")
print(" authenticated!")
#print(" authenticated!")
return token
else:
print(f"Authentication failed. Status code: {response.status_code}")
@@ -36,42 +36,126 @@ def send_patch_request(device_ip, token, connection_string):
else:
print(f"Failed to send PATCH request to device {device_ip}. Status code: {response.status_code}")
# Function to send PATCH request
def patch_time(device_ip, token):
headers = {
"mx-api-token": token
}
# payload = {
# "ntp": {
# "source": "timeserver",
# "server": "10.84.171.254",
# "enable": True
# }
# }
payload = {
"timezone": "America/Chicago"
}
patch_url = f"https://{device_ip}:8443/api/v1/device/time"
response = requests.patch(patch_url, json=payload, headers=headers, verify=False)
if response.status_code == 200:
json_data = json.loads(response.content.decode())
time = json_data['data']['time']
timezone = json_data['data']['timezone']
last = json_data['data']['lastUpdateTime']
server = json_data['data']['ntp']['server']
enabled = json_data['data']['ntp']['enable']
print(time + " " + timezone + " " + last + " " + server + " " + str(enabled))
else:
json_data = json.loads(response.content.decode())
#print(json.dumps(json_data, indent=2))
# Function to send UPGRADE request
def send_upgrade_request(device_ip, token, upgrade_url):
headers = {
"mx-api-token": token
}
payload = {
"deleteFileAfterInstallComplete": True,
"download": True,
"install": True,
"url": upgrade_url
"url": upgrade_url,
}
patch_url = f"https://{device_ip}:8443/api/v1/upgrades"
response = requests.post(patch_url, json=payload, headers=headers, verify=False)
if response.status_code == 200:
print(f"POST request successful for device {device_ip}")
else:
print(f"Failed to send POST request to device {device_ip}. Status code: {response.status_code}")
print(response.content)
json_data = json.loads(response.content.decode())
id = json_data['data']['id']
return id
# Function to send UPGRADE request
def get_upgrade_job(device_ip, token, id):
headers = {
"mx-api-token": token
}
payload = {
}
patch_url = f"https://{device_ip}:8443/api/v1/upgrades/{id}"
response = requests.get(patch_url, json=payload, headers=headers, verify=False)
if response.status_code == 200:
#print(f"GET request successful for device {device_ip}")
#print(json_str)
#print(json_data['data'][json_data['count'] - 1]['parameter']['url'], json_data['data'][json_data['count'] - 1]['state'], json_data['count'])
json_data = json.loads(response.content.decode())
getid = json_data['data']['id']
created = json_data['data']['createdAt']
cur_status = json_data['data']['state']
current_tasks = json_data['data']['completedTask']
total_tasks = json_data['data']['totalTask']
print("JOB #" + str(getid) + " " + str(current_tasks) + "/" + str(total_tasks))
print("CREATED ON: " + str(created))
print("CURRENT STATUS: " + str(cur_status))
print(json.dumps(json_data, indent=2))
else:
print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}")
print(response.content)
def get_upgrade_jobs(device_ip, token):
headers = {
"mx-api-token": token
}
payload = {
}
patch_url = f"https://{device_ip}:8443/api/v1/upgrades/2"
patch_url = f"https://{device_ip}:8443/api/v1/upgrades"
response = requests.get(patch_url, json=payload, headers=headers, verify=False)
if response.status_code == 200:
#print(f"GET request successful for device {device_ip}")
#print(json_str)
#print(json_data['data'][json_data['count'] - 1]['parameter']['url'], json_data['data'][json_data['count'] - 1]['state'], json_data['count'])
json_data = json.loads(response.content.decode())
json_str = json.dumps(json_data)
print(jq.compile('.data.completedAt').input(json.loads(json_str)).first())
count = json_data['count']
print(str(count))
for i in range(count):
getid = json_data['data'][i]['id']
created = json_data['data'][i]['createdAt']
cur_status = json_data['data'][i]['state']
current_tasks = json_data['data'][i]['completedTask']
total_tasks = json_data['data'][i]['totalTask']
print("JOB #" + str(getid) + " " + str(current_tasks) + "/" + str(total_tasks))
print("CREATED ON: " + str(created))
print("CURRENT STATUS: " + str(cur_status))
else:
print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}")
print(response.content)
# Function to send UPGRADE request
def start_upgrade_job(device_ip, token, id):
headers = {
"mx-api-token": token
}
payload = {
}
patch_url = f"https://{device_ip}:8443/api/v1/upgrades/{id}/start"
response = requests.put(patch_url, json=payload, headers=headers, verify=False)
if response.status_code == 200:
json_data = json.loads(response.content.decode())
curid = json_data['data']['id']
startedat = json_data['data']['startedAt']
print("Job #" + str(curid) + " started on " + str(startedat))
else:
print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}")
print(response.content.decode())
# Function to send UPGRADE request
def put_API(device_ip, token):
headers = {
@@ -79,14 +163,15 @@ def put_API(device_ip, token):
}
payload = {
}
patch_url = f"https://{device_ip}:8443/api/v1/azure-iotedge/reset"
patch_url = f"https://{device_ip}:8443/api/v1/upgrades/3/start"
response = requests.put(patch_url, json=payload, headers=headers, verify=False)
if response.status_code == 200:
#print(f"GET request successful for device {device_ip}")
#json_data = json.loads(response.content.decode())
#json_str = json.dumps(json_data)
#print(jq.compile('.data.completedAt').input(json.loads(json_str)).first())
print(response.content.decode())
json_data = json.loads(response.content.decode())
print(json_data['data']['firmwareVersion'])
else:
print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}")
print(response.content.decode())
@@ -97,51 +182,121 @@ def get_API(device_ip, token):
}
payload = {
}
patch_url = f"https://{device_ip}:8443/api/v1/azure-iotedge"
patch_url = f"https://{device_ip}:8443/api/v1/device/general"
response = requests.get(patch_url, json=payload, headers=headers, verify=False)
if response.status_code == 200:
#print(f"GET request successful for device {device_ip}")
#json_data = json.loads(response.content.decode())
#json_str = json.dumps(json_data)
#print(jq.compile('.data.completedAt').input(json.loads(json_str)).first())
print(response.content.decode())
json_data = json.loads(response.content.decode())
hostname = json_data['data']['hostName']
serial = json_data['data']['serialNumber']
version = json_data['data']['firmwareVersion']
description = json_data['data']['description']
print(hostname + " " + serial + " " + version + " " + description)
#json_str = json.dumps(json_data, indent=2)
#print(json_str)
else:
print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}")
print(response.content.decode())
def get_time(device_ip, token):
headers = {
"mx-api-token": token
}
payload = {
}
patch_url = f"https://{device_ip}:8443/api/v1/device/time"
response = requests.get(patch_url, json=payload, headers=headers, verify=False)
if response.status_code == 200:
#print(f"GET request successful for device {device_ip}")
#json_data = json.loads(response.content.decode())
#json_str = json.dumps(json_data)
#print(jq.compile('.data.completedAt').input(json.loads(json_str)).first())
json_data = json.loads(response.content.decode())
time = json_data['data']['time']
timezone = json_data['data']['timezone']
last = json_data['data']['lastUpdateTime']
print(time + " " + timezone + " " + last)
#json_str = json.dumps(json_data, indent=2)
#print(json_str)
else:
print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}")
print(response.content.decode())
def delete_API(device_ip, token):
headers = {
"mx-api-token": token
}
payload = {
}
patch_url = f"https://{device_ip}:8443/api/v1/upgrades/5"
response = requests.delete(patch_url, json=payload, headers=headers, verify=False)
print(response.status_code)
json_data = json.loads(response.content.decode())
json_str = json.dumps(json_data, indent=2)
print(json_str)
# Read the Excel file
excel_file_path = ""
if excel_file_path == "":
print("Provide Excel file path before running the script!")
exit(11)
df = pd.read_excel(excel_file_path)
df = df[df["device_name"].notnull()]
# excel_file_path = ""
# if excel_file_path == "":
# print("Provide Excel file path before running the script!")
# exit(11)
# df = pd.read_excel(excel_file_path)
# df = df[df["device_name"].notnull()]
# Iterate over each row in the DataFrame
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
for index, row in df.iterrows():
device_name = row['device_name']
device_ip_address_https = row['device_ip_address_http']
connection_string = row['connection_string']
upgrade_url = "" #https://files.thingsprocloud.com/package/moxa-aig-301-series-includes-security-patch-firmware-v1.0.deb.yaml
if upgrade_url == "":
print("Provide upgrade URL before running the script!")
exit(12)
# for index, row in df.iterrows():
# device_name = row['device_name']
# device_ip_address_https = row['device_ip_address_http']
# connection_string = row['connection_string']
# upgrade_url = "" #https://files.thingsprocloud.com/package/moxa-aig-301-series-includes-security-patch-firmware-v1.0.deb.yaml
# if upgrade_url == "":
# print("Provide upgrade URL before running the script!")
# exit(12)
# Authenticate and get token
# # Authenticate and get token
# payload_auth = {
# "acceptEULA": True,
# "name": "",
# "password": ""
# }
# if payload_auth["name"] == "" or payload_auth["password"] == "":
# print("Provide the credentials before running the script!")
# exit(10)
# print(device_name, end="")
# token = authenticate(device_ip_address_https, payload_auth)
# if token:
# get_API(device_ip_address_https, token)
# print("\n")
for i in range(193, 222):
upgrade_url = "https://files.thingsprocloud.com/package/Upgrade_AIG-301_2.4.0-4020_IMG_1.4_to_1.5.deb.yaml"
payload_auth = {
"acceptEULA": True,
"name": "",
"password": ""
"name": "admin",
"password": "admin@123"
}
if payload_auth["name"] == "" or payload_auth["password"] == "":
print("Provide the credentials before running the script!")
exit(10)
print(device_name, end="")
token = authenticate(device_ip_address_https, payload_auth)
device_ip_address = str("10.84.171." + str(i))
#print(device_ip_address, end="")
token = authenticate(device_ip_address, payload_auth)
if token:
get_API(device_ip_address_https, token)
print("\n")
#id = send_upgrade_request(device_ip_address,token,upgrade_url)
#print(id)
#get_upgrade_job(device_ip_address, token, 6)
#start_upgrade_job(device_ip_address, token, id)
#put_API(device_ip_address, token)
#patch_time(device_ip_address,token)
#get_time(device_ip_address, token)
#delete_API(device_ip_address,token)
get_API(device_ip_address, token)
else:
print("Authentication failed!")

View File

@@ -1,7 +1,6 @@
import pandas as pd
import requests
from urllib3.exceptions import InsecureRequestWarning
import jq
import json
import scp
import paramiko
@@ -68,37 +67,46 @@ def ssh_execute_command_with_password(hostname, username, password, command):
ssh_client.close()
# Read the Excel file
excel_file_path = ""
if excel_file_path == "":
print("Provide Excel file path before running the script!")
exit(11)
df = pd.read_excel(excel_file_path)
df = df[df["device_name"].notnull()]
# excel_file_path = ""
# if excel_file_path == "":
# print("Provide Excel file path before running the script!")
# exit(11)
# df = pd.read_excel(excel_file_path)
# df = df[df["device_name"].notnull()]
# Iterate over each row in the DataFrame
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
local_file_path = ""#"./azureiotedge_2.4.0-2697_armhf.mpkg"
local_file_path = "AIG-301_1.5.2-20240625_saft1_armhf.deb"
if local_file_path == "":
print("Provide upgrade file path before running the script!")
exit(12)
remote_file_path = "./."
username = ""
password = ""
username = "moxa"
password = "moxa"
if username == "" or password == "":
print("Provide credentials before running the script!")
exit(10)
command = ""#"appman app install azureiotedge_2.4.0-2697_armhf.mpkg"
if command == "":
if command == "dpkg -i AIG-301_1.5.2-20240625_saft1_armhf.deb":
print("Provide a command to execute before running the script!")
exit(11)
for index, row in df.iterrows():
device_name = row['device_name']
device_ip_address_https = row['device_ip_address_http']
print(device_name)
ssh_execute_command_with_password(device_ip_address_https, username, password, command)
print("\n")
# for index, row in df.iterrows():
# device_name = row['device_name']
# device_ip_address_https = row['device_ip_address_http']
# print(device_name)
# #ssh_execute_command_with_password(device_ip_address_https, username, password, command)
# print("\n")
for i in range(131, 160):
device_ip_address = str("10.84.157." + str(i))
print(device_ip_address, end="")
if i == 136 or i == 138 or i == 151:
print(" DONE")
else:
print(" TODO")
scp_file(local_file_path, remote_file_path, device_ip_address, username, password)

View File

@@ -0,0 +1,28 @@
import re
# Function to transform each line
def transform_line(line):
# Use regex to extract parts of the line
match = re.match(r'"([^"]+)","([^"]+)","\[(.*)\]"', line)
if match:
id_part = match.group(1)
date_part = match.group(2)
categories_part = match.group(3)
# Remove extra quotes and split the categories
categories = categories_part.replace('""', '"').split(',')
# Swap categories order and join them with a semicolon
transformed_categories = ','.join(categories[::-1])
# Return the transformed line
return f'"{id_part}",{transformed_categories}'
else:
return None
# Open input file and output file
with open('export_hierarchy', 'r') as infile, open('export_hierarchy_transformed', 'w') as outfile:
for line in infile:
transformed_line = transform_line(line.strip())
if transformed_line:
outfile.write(transformed_line + '\n')