Merge branch 'feature/cli_claim_support' into 'master'

Feature/cli claim support

See merge request app-frameworks/esp-rainmaker!89
This commit is contained in:
Piyush Shah
2020-04-27 18:28:35 +08:00
10 changed files with 500 additions and 29 deletions

View File

@@ -16,7 +16,8 @@
import argparse
from rmaker_cmd.node import get_nodes, get_node_config, get_node_status,\
set_params, get_params, remove_node, get_mqtt_host
set_params, get_params, remove_node,\
get_mqtt_host, claim_node
from rmaker_cmd.user import signup, login, forgot_password
from rmaker_cmd.provision import provision
from rmaker_cmd.test import test
@@ -82,6 +83,7 @@ def main():
help='Node ID for the node')
setparams_parser = setparams_parser.add_mutually_exclusive_group(
required=True)
setparams_parser.add_argument('--filepath',
help='Path of the JSON file\
containing parameters to be set')
@@ -122,6 +124,13 @@ def main():
firmware')
getmqtthost_parser.set_defaults(func=get_mqtt_host)
claim_parser = subparsers.add_parser('claim',
help='Claim the node connected to the given serial port\
(Get cloud credentials)')
claim_parser.add_argument("port", metavar='<port>',
help='Serial Port connected to the device.')
claim_parser.set_defaults(func=claim_node)
test_parser = subparsers.add_parser('test',
help='Test commands to check\
user node mapping')

View File

@@ -21,6 +21,7 @@ try:
from rmaker_lib import session, node, serverconfig, configmanager
from rmaker_lib.exceptions import NetworkError, InvalidJSONError, SSLError
from rmaker_lib.logger import log
from rmaker_tools.rmaker_claim.claim import claim
except ImportError as err:
print("Failed to import ESP Rainmaker library. " + str(err))
raise err
@@ -230,7 +231,7 @@ def get_mqtt_host(vars=None):
log.debug("Get MQTT Host request url : " + request_url)
response = requests.get(url=request_url,
verify=configmanager.CERT_FILE)
log.debug("Get MQTT Host resonse : " + response.text)
log.debug("Get MQTT Host response : " + response.text)
response.raise_for_status()
except requests.exceptions.SSLError:
raise SSLError
@@ -250,3 +251,17 @@ def get_mqtt_host(vars=None):
else:
log.error("MQTT Host does not exists.")
return response['mqtt_host']
def claim_node(vars=None):
"""
Claim the node connected to the given serial port
(Get cloud credentials)
:param args:
a) port - Serial Port connected to the device
"""
try:
claim(vars['port'])
except Exception as claim_err:
log.error(claim_err)
return

View File

@@ -32,7 +32,6 @@ PROVISION_FAILURE_MSG = 'Provisioning Failed. Reset your board to factory'
'defaults and retry.'
def provision(vars=None):
"""
Provisioning of the node.

View File

@@ -49,7 +49,7 @@ def signup(vars=None):
log.error(signup_err)
else:
if status is True:
verification_code = input('Enter verification code sent on your'
verification_code = input('Enter verification code sent on your '
'Email.\n Verification Code : ')
try:
status = u.signup(verification_code)

View File

@@ -82,13 +82,10 @@ class Config:
raise set_config_err
log.info("Configured config file successfully.")
def get_config(self, node=None, config_file=CONFIG_FILE):
def get_config(self, config_file=CONFIG_FILE):
"""
Get the configuration details from config file.
:params node: Name of node
:type node: str
:params config_file: Config filename to read config data from
:type data: str
@@ -103,31 +100,22 @@ class Config:
"""
file = Path(path.expanduser(HOME_DIRECTORY) + config_file)
if not file.exists():
if node and node.lower() == 'esp32':
return None
else:
raise InvalidUserError
raise InvalidUserError
try:
with open(path.join(path.expanduser(HOME_DIRECTORY),
config_file), 'r') as config_file:
data = json.load(config_file)
if node and node.lower() == "esp32":
return data
else:
idtoken = data['idtoken']
refresh_token = data['refreshtoken']
access_token = data['accesstoken']
idtoken = data['idtoken']
refresh_token = data['refreshtoken']
access_token = data['accesstoken']
except Exception as get_config_err:
raise get_config_err
return idtoken, refresh_token, access_token
def get_binary_config(self, node=None, config_file=CONFIG_FILE):
def get_binary_config(self, config_file=CONFIG_FILE):
"""
Get the configuration details from binary config file.
:params node: Name of node
:type node: str
:params config_file: Config filename to read config data from
:type data: str
@@ -139,14 +127,12 @@ class Config:
"""
file = Path(path.expanduser(HOME_DIRECTORY) + config_file)
if not file.exists():
if node and node.lower() == 'esp32':
return None
return None
try:
with open(path.join(path.expanduser(HOME_DIRECTORY),
config_file), 'rb') as config_file:
data = config_file.read()
if node and node.lower() == "esp32":
return data
return data
except Exception as get_config_err:
raise get_config_err
return

View File

@@ -25,7 +25,7 @@ log_filename = "logs/log_" + date_time_obj.strftime("%d-%m-%Y") + ".log"
log = logging.getLogger("CLI_LOGS")
file_formatter = logging.Formatter('%(asctime)s:[%(funcName)s]:\
[%(levelname)s]:%(message)s')
[%(levelname)s]:%(message)s')
console_formatter = logging.Formatter('[%(levelname)s]:%(message)s')
log.setLevel(logging.DEBUG)

View File

@@ -34,7 +34,7 @@ class Session:
self.id_token = config.get_access_token()
if self.id_token is None:
raise InvalidConfigError
self.request_header = {'content-type': 'application/json',
self.request_header = {'Content-Type': 'application/json',
'Authorization': self.id_token}
def get_nodes(self):
@@ -90,7 +90,7 @@ class Session:
log.debug("Get MQTT Host request url : " + request_url)
response = requests.get(url=request_url,
verify=configmanager.CERT_FILE)
log.debug("Get MQTT Host resonse : " + response.text)
log.debug("Get MQTT Host response : " + response.text)
response.raise_for_status()
except requests.exceptions.SSLError:
raise SSLError

View File

@@ -0,0 +1,13 @@
# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

View File

@@ -0,0 +1,432 @@
# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from os import path
from pathlib import Path
from io import StringIO
import sys
from sys import exit
import time
import requests
import json
import binascii
from types import SimpleNamespace
from rmaker_lib.logger import log
from cryptography import x509
from cryptography.x509.oid import NameOID
from cryptography.hazmat.primitives import hashes, hmac, serialization
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa
from rmaker_tools.rmaker_claim.claim_config import \
CLAIM_INITIATE_URL, CLAIM_VERIFY_URL
from rmaker_lib import session, configmanager
from rmaker_lib.exceptions import SSLError
from rmaker_cmd import node
if os.getenv('IDF_PATH'):
sys.path.insert(0, os.path.join(os.getenv('IDF_PATH'),
'components',
'esptool_py',
'esptool'))
sys.path.insert(0, os.path.join(os.getenv('IDF_PATH'),
'components',
'nvs_flash',
'nvs_partition_generator'))
import esptool
import nvs_partition_gen
else:
log.error("Please set the IDF_PATH environment variable.")
exit(0)
CERT_FILE = './server_cert/server_cert.pem'
# List of efuse blocks
#
# Name, Index, Read Address, Read Protect Bit, Write Protect Bit
BLOCKS = [
("BLOCK_SYS_DATA", 2, 0x3f41a05c, None, 21)
]
def flash_bin_onto_node(port, esptool, bin_to_flash):
try:
command = ['--port', port, 'write_flash', '0x340000', bin_to_flash]
esptool.main(command)
except Exception as err:
log.error(err)
sys.exit(1)
def get_node_platform_and_mac(esptool, port):
sys.stdout = mystdout = StringIO()
command = ['--port', port, 'chip_id']
log.info("Running esptool command to get node\
platform and mac from device")
esptool.main(command)
sys.stdout = sys.__stdout__
# Finding chip type from output.
node_platform = next(filter(lambda line: 'Detecting chip type' in line,
mystdout.getvalue().splitlines()))
# Finds the first occurence of the line
# with the MAC Address from the output.
mac = next(filter(lambda line: 'MAC: ' in line,
mystdout.getvalue().splitlines()))
mac_addr = mac.split('MAC: ')[1].replace(':', '').upper()
platform = node_platform.split()[-1].lower()
return platform, mac_addr
def get_secret_key(port, esptool):
esp = esptool.ESP32ROM(port)
esp.connect('default_reset')
for (name, idx, read_addr, _, _) in BLOCKS:
addrs = range(read_addr, read_addr + 32, 4)
secret = "".join(["%08x" % esp.read_reg(addr) for addr in addrs[0:4]])
secret = secret[6:8]+secret[4:6]+secret[2:4]+secret[0:2] +\
secret[14:16]+secret[12:14]+secret[10:12]+secret[8:10] +\
secret[22:24]+secret[20:22]+secret[18:20]+secret[16:18] +\
secret[30:32]+secret[28:30]+secret[26:28]+secret[24:26]
return secret
def gen_hmac_challenge_resp(secret_key, hmac_challenge):
h = hmac.HMAC(bytes.fromhex(secret_key),
hashes.SHA512(),
backend=default_backend())
h.update(bytes(hmac_challenge, 'utf-8'))
hmac_challenge_response = binascii.hexlify(h.finalize()).decode()
return hmac_challenge_response
def gen_host_csr(private_key, common_name=None):
# Generate CSR on host
builder = x509.CertificateSigningRequestBuilder()
builder = builder.subject_name(x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, common_name),
]))
builder = builder.add_extension(
x509.BasicConstraints(ca=False, path_length=None), critical=True,
)
request = builder.sign(
private_key, hashes.SHA256(), default_backend()
)
if isinstance(request, x509.CertificateSigningRequest) is not True:
print("Certificate Signing Request could not be created")
return None
csr = request.public_bytes(serialization.Encoding.PEM).decode("utf-8")
return csr
def create_files_of_claim_info(dest_filedir, node_id, private_key, node_cert,
endpointinfo, node_info_csv):
try:
log.debug("Writing node info at location: " + dest_filedir +
'node.info')
# Create files for each claim data - node info, node key,
# node cert, endpoint info
with open(dest_filedir+'node.info', 'w+') as info_file:
info_file.write(node_id)
log.debug("Writing node info at location: " +
dest_filedir + 'node.key')
with open(dest_filedir+'node.key', 'wb+') as info_file:
info_file.write(private_key)
log.debug("Writing node info at location: " +
dest_filedir + 'node.crt')
with open(dest_filedir+'node.crt', 'w+') as info_file:
info_file.write(node_cert)
log.debug("Writing node info at location: " +
dest_filedir + 'endpoint.info')
with open(dest_filedir+'endpoint.info', 'w+') as info_file:
info_file.write(endpointinfo)
log.debug("Writing node info at location: " +
dest_filedir + 'node_info.csv')
with open(dest_filedir+'node_info.csv', 'w+') as info_file:
for input_line in node_info_csv:
info_file.write(input_line)
info_file.write("\n")
except Exception as file_error:
raise file_error
def claim(port):
try:
node_id = None
node_info = None
hmac_challenge = None
claim_verify_data = None
claim_initiate_url = CLAIM_INITIATE_URL
claim_verify_url = CLAIM_VERIFY_URL
private_key = None
curr_claim_data = None
user_whitelist_err_msg = ('user is not allowed to claim esp32 device.'
' please contact administrator')
config = configmanager.Config()
userid = config.get_user_id()
creds_dir = Path(path.expanduser(
str(Path(path.expanduser(configmanager.HOME_DIRECTORY))) +
'/' +
str(Path(path.expanduser(
configmanager.CONFIG_DIRECTORY))) +
'/claim_data/' +
userid
))
if not creds_dir.exists():
os.makedirs(path.expanduser(creds_dir))
log.debug("Creating new directory " + str(creds_dir))
print("\nClaiming process started. This may take time.")
log.info("Claiming process started. This may take time.")
node_platform, mac_addr = get_node_platform_and_mac(esptool, port)
print("Node platform detected is: ", node_platform)
print("MAC address is: ", mac_addr)
log.debug("MAC address received: " + mac_addr)
log.debug("Node platform detected is: " + node_platform)
log.info("Creating session")
curr_session = session.Session()
header = curr_session.request_header
start = time.time()
mac_dir = Path(path.expanduser(str(creds_dir) + '/' + mac_addr))
if not mac_dir.exists():
os.makedirs(path.expanduser(mac_dir))
log.debug("Creating new directory " + str(mac_dir))
output_bin_filename = mac_addr + '.bin'
mac_dir_path = str(mac_dir) + '/'
# Set values
dest_filedir = mac_dir_path
# Set csv file data
node_info_csv = [
'key,type,encoding,value',
'rmaker_creds,namespace,,',
'node_id,file,binary,' +
dest_filedir + 'node.info',
'mqtt_host,file,binary,' +
dest_filedir + 'endpoint.info',
'client_cert,file,binary,' +
dest_filedir + 'node.crt',
'client_key,file,binary,' +
dest_filedir + 'node.key'
]
# Generate nvs args to be sent to NVS Partition Utility
nvs_args = SimpleNamespace(input=dest_filedir+'node_info.csv',
output=output_bin_filename,
size='0x6000',
outdir=dest_filedir,
version=2)
# Set config mac addr path
mac_addr_config_path = str(Path(path.expanduser(
configmanager.CONFIG_DIRECTORY))) + '/claim_data/' +\
mac_addr +\
'/' + output_bin_filename
# Check if claim data for node exists in CONFIG directory
log.debug("Checking if claim data for node exists in directory: " +
configmanager.CONFIG_DIRECTORY)
curr_claim_data = configmanager.Config().get_binary_config(
config_file=mac_addr_config_path)
if curr_claim_data:
print("\nClaiming data already exists at location: " +
dest_filedir)
log.debug("Claiming data already exists at location: " +
dest_filedir)
log.info("Using existing claiming data")
print("Using existing claiming data")
print("Generating NVS Partition binary: " + dest_filedir +
output_bin_filename)
# Run NVS Partition Utility to create binary of node info data
# and flash onto node
log.debug("Generating NVS Partition binary: " + dest_filedir +
output_bin_filename)
nvs_partition_gen.generate(nvs_args)
print("\nFlashing binary onto node\n")
log.info("Flashing binary onto node")
flash_bin_onto_node(port, esptool, dest_filedir +
output_bin_filename)
log.info("Binary flashed onto node")
return
# Generate Key
log.info("Generate RSA key")
private_key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
)
log.info("RSA Private Key generated")
# Set Claim initiate request data
claim_initiate_data = {"mac_addr": mac_addr, "platform": node_platform}
claim_init_enc_data = str(claim_initiate_data).replace(
"'", '"')
print("Claim initiate started")
# Sign the CSR using the CA
try:
# Claim Initiate Request
log.info("Claim initiate started. Sending claim/initiate POST\
request")
log.debug("Claim Initiate POST Request: url: " +
claim_initiate_url + "data: " +
str(claim_init_enc_data) +
"headers: " + str(header) +
"verify: " + CERT_FILE)
claim_initiate_response = requests.post(url=claim_initiate_url,
data=claim_init_enc_data,
headers=header,
verify=CERT_FILE)
if claim_initiate_response.status_code != 200:
log.error("Claim initiate failed.\n" +
claim_initiate_response.text)
exit(0)
print("Claim initiate done")
log.debug("Claim Initiate POST Response: status code: " +
str(claim_initiate_response.status_code) +
" and response text: " + claim_initiate_response.text)
log.info("Claim initiate done")
# Get data from response depending on node_platform
if node_platform == "esp32":
# Generate CSR with common_name=node_id received in response
node_id = str(json.loads(
claim_initiate_response.text)['node_id'])
print("Generating CSR")
log.info("Generating CSR")
csr = gen_host_csr(private_key, common_name=node_id)
if not csr:
raise Exception("CSR Not Generated. Claiming Failed")
log.info("CSR generated")
claim_verify_data = {"csr": csr}
# Save node id as node info to use while saving claim data
# in csv file
node_info = node_id
else:
auth_id = str(json.loads(
claim_initiate_response.text)['auth_id'])
hmac_challenge = str(json.loads(
claim_initiate_response.text)['challenge'])
print("Generating CSR")
log.info("Generating CSR")
csr = gen_host_csr(private_key, common_name=mac_addr)
if not csr:
raise Exception("CSR Not Generated. Claiming Failed")
log.info("CSR generated")
log.info("Getting secret key from device")
secret_key = get_secret_key(port, esptool)
log.info("Getting secret key from device")
log.info("Generating hmac challenge response")
hmac_challenge_response = gen_hmac_challenge_resp(
secret_key,
hmac_challenge)
hmac_challenge_response = hmac_challenge_response.strip('\n')
log.debug("Secret Key generated: " + secret_key)
log.debug("HMAC Challenge Response: " +
hmac_challenge_response)
claim_verify_data = {"auth_id":
auth_id,
"challenge_response":
hmac_challenge_response,
"csr":
csr}
# Save node id as node info to use while saving claim data
# in csv file
node_info = mac_addr
claim_verify_enc_data = str(claim_verify_data).replace(
"'", '"')
log.debug("Claim Verify POST Request: url: " + claim_verify_url +
"data: " + str(claim_verify_enc_data) + "headers: " +
str(header) + "verify: " + CERT_FILE)
claim_verify_response = requests.post(url=claim_verify_url,
data=claim_verify_enc_data,
headers=header,
verify=CERT_FILE)
if claim_verify_response.status_code != 200:
claim_verify_response_json = json.loads(
claim_verify_response.text.lower())
if (claim_verify_response_json["description"] in
user_whitelist_err_msg):
log.error('Claim verification failed.\n' +
claim_verify_response.text)
print('\nYour account isn\'t whitelisted for ESP32.'
' Please send your registered email address to'
' esp-rainmaker-admin@espressif.com for whitelisting'
)
exit(0)
print("Claim verify done")
log.debug("Claim Verify POST Response: status code: " +
str(claim_verify_response.status_code) +
" and response text: " + claim_verify_response.text)
log.info("Claim verify done")
node_cert = json.loads(claim_verify_response.text)['certificate']
print("Claim certificate received")
log.info("Claim certificate received")
except requests.exceptions.SSLError:
raise SSLError
except requests.ConnectionError:
log.error("Please check the Internet connection.")
exit(0)
# Set node claim data
sys.stdout = StringIO()
log.info("Getting MQTT Host")
endpointinfo = node.get_mqtt_host(None)
log.debug("Endpoint info received: " + endpointinfo)
sys.stdout = sys.__stdout__
# Extract private key in bytes from private key object generated
log.info("Extracting private key in bytes")
node_private_key = private_key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption())
# Create files of each claim data info
print("\nSaving claiming data info at location: ", dest_filedir)
log.debug("Saving claiming data info at location: " +
dest_filedir)
create_files_of_claim_info(dest_filedir, node_info, node_private_key,
node_cert, endpointinfo, node_info_csv)
# Run NVS Partition Utility to create binary of node info data
print("\nGenerating NVS Partition Binary from claiming data: " +
dest_filedir + output_bin_filename)
log.debug("Generating NVS Partition Binary from claiming data: " +
dest_filedir + output_bin_filename)
nvs_partition_gen.generate(nvs_args)
print("\nFlashing onto node\n")
log.info("Flashing binary onto node")
flash_bin_onto_node(port, esptool, dest_filedir + output_bin_filename)
print("Claiming done")
log.info("Claiming done")
print("Time(s):" + str(time.time() - start))
except Exception as err:
log.error(err)
sys.exit(err)

View File

@@ -0,0 +1,17 @@
# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
CLAIM_BASE_URL = "https://esp-claiming.rainmaker.espressif.com/"
CLAIM_INITIATE_URL = CLAIM_BASE_URL+"claim/initiate"
CLAIM_VERIFY_URL = CLAIM_BASE_URL+"claim/verify"