From ade23d85eeab4bf9f2338d58113718d34f9b69fb Mon Sep 17 00:00:00 2001 From: XinYi Song <2037158277@qq.com> Date: Tue, 15 Feb 2022 15:48:05 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=86=E5=BD=92=E6=A1=A3?= =?UTF-8?q?=E9=81=A5=E6=84=9F=E6=95=B0=E6=8D=AE=E7=9A=84=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- util/file_util.py | 1009 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1009 insertions(+) create mode 100644 util/file_util.py diff --git a/util/file_util.py b/util/file_util.py new file mode 100644 index 0000000..643ca53 --- /dev/null +++ b/util/file_util.py @@ -0,0 +1,1009 @@ +""" +Author : XinYi Song +Time : 2022/2/12 17:13 +Desc: +""" +import os +import time + +import requests + +from application.settings import Config +from common.tools.dms import dms_login, dms_task_record, dms_sensing_data +from scan_data.GetMetaInfo import GetGFPMSData, GetGF3MDJData, GetH08Data, GetSentinel1Data, GetSentinel2Data, \ + GetSentinel3OLData, GetHJ1Data, GetZY3Data, GetSNPPData +from scan_data.example import GetJPSSData +from upload.upload_client import upload_file_client +from util.http_file_upload import upload_client +from util.http_util import httpUtil +from util.remote_sensing_util import gf4_pmi_001 +from util.xml_util import dict_to_xml +from apscheduler.schedulers.blocking import BlockingScheduler + +sched = BlockingScheduler() + + +def list_dir(file_dir): + """ + 通过 listdir 得到的是仅当前路径下的文件名，不包括子目录中的文件，如果需要得到所有文件需要递归 + """ + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return + fileNameList = [] + dms_list = dms_sensing_data(token_s) + for dms in dms_list: + fileNameList.append(dms['fileName']) + dir_list = os.listdir(file_dir) + # 判断扫描出的文件和已有的文件，将多出的文件进行解析 + d = [y for y in dir_list if y not in fileNameList] + if d is None or len(d) == 0: + print('没有多余的遥感数据文件，终止程序') + return + file_total_size = "" + file_total_name = "" + xmlPath = '' # 解析出的xml文件保存的路径 + ThumbnailPath = '' # 解析出的缩略图保存的路径 + for cur_file in d: + # 获取文件的绝对路径 + path = os.path.join(file_dir, cur_file) + if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 + file_name = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.tiff' + xml_path = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml' + # 解析遥感数据文件（demo） + gf4_sensing = gf4_pmi_001(file_name, xml_path) + file_name = os.path.basename(path).split('.')[0] + xml_path = 'D:/file/work/pythonyuanma/dms_management/xml/' + file_name + '.xml' + # 将遥感数据写入xml中，并保存本地文件夹中 + dict_to_xml(gf4_sensing, xml_path) + + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(xml_path, 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项 + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(gf4_sensing['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项 + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = gf4_sensing['CollectionCode'] + DirectoryDepth = gf4_sensing['DirectoryDepth'] + StartTime = gf4_sensing['StartTime'] + uc = upload_client(path, DirectoryDepth, StartTime) + + StartTime = time.mktime(time.strptime(gf4_sensing['StartTime'], '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime(time.strptime(gf4_sensing['EndTime'], '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": "GF4_PMI_001", "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": gf4_sensing['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": gf4_sensing['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + # print("========"+suffix) + print("{0} : is file!".format(cur_file)) + if os.path.isdir(path): + print("{0} : is dir!".format(cur_file)) + list_dir(path) # 递归子目录 + if uc['type'] == 'ok': + continue + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": "GF4_PMI_001", "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + + +def scan_VJ102_dir(collectionCode, path): + """ + 解析JPSS-VJ102元数据 + :return: + """ + print('开始扫描VJ102IMG数据集') + #collectionCode = 'VJ102IMG' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collectionCode) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return + + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath) + + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(JPSSData_dict['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = JPSSData_dict['CollectionCode'] + DirectoryDepth = JPSSData_dict['DirectoryDepth'] + StartTime = JPSSData_dict['StartTime'] + # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) + uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) + + StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + + +def scan_VJ103_dir(collectionCode, path): + """ + 解析JPSS-VJ103元数据 + :param file_dir: + :return: + """ + # file_dir = 'E:/数管' + file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\JPSS' + print('开始扫描VJ103IMG数据集') + #collectionCode = 'VJ103IMG' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collectionCode) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath) + + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(JPSSData_dict['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = JPSSData_dict['CollectionCode'] + DirectoryDepth = JPSSData_dict['DirectoryDepth'] + StartTime = JPSSData_dict['StartTime'] + # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) + uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) + + StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + # print("========"+suffix) + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + + +def scan_GF1_PMS2_dir(collectionCode, path): + """ + 获取高分 PMS卫星元数据 + :return: + """ + + print('开始扫描GF1_PMS2_001数据集') + #collectionCode = 'GF1_PMS2_001' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collectionCode) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + GFPMS_dict = GetGFPMSData(path, xmlPath, ThumbnailPath) + + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + print(GFPMS_dict['xmlPath']) + files = {'file': open(GFPMS_dict['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(GFPMS_dict['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = GFPMS_dict['CollectionCode'] + DirectoryDepth = GFPMS_dict['DirectoryDepth'] + StartTime = GFPMS_dict['StartTime'] + # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) + uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) + + StartTime = time.mktime(time.strptime(GFPMS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime(time.strptime(GFPMS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": GFPMS_dict['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": GFPMS_dict['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + + +def scan_GF3MDJ_dir(collectionCode, path): + """ + 获取高分3号MDJ（GF-3 MDJ）卫星元数据 + :return: + """ + # file_dir = 'E:/数管' + print('开始扫描GF3_MDJ_SS数据集') + #collectionCode = 'GF3_MDJ_SS' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collectionCode) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + GF3_MDJ_SS_dict = GetGF3MDJData(path, xmlPath, ThumbnailPath) + + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(GF3_MDJ_SS_dict['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(GF3_MDJ_SS_dict['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = GF3_MDJ_SS_dict['CollectionCode'] + DirectoryDepth = GF3_MDJ_SS_dict['DirectoryDepth'] + StartTime = GF3_MDJ_SS_dict['StartTime'] + # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) + uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) + + StartTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": GF3_MDJ_SS_dict['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": GF3_MDJ_SS_dict['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + # print("========"+suffix) + + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + + +def scan_H08_dir(collectionCode, path): + """ + 获取葵花8卫星元数据 + :return: + """ + print('开始扫描H08数据集') + #collectionCode = 'NC_H08' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collectionCode) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return + + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + GetH08_dict = GetH08Data(path, xmlPath, ThumbnailPath) + + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(GetH08_dict['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(GetH08_dict['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = GetH08_dict['CollectionCode'] + DirectoryDepth = GetH08_dict['DirectoryDepth'] + StartTime = GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19] + # uc = upload_client(path, DirectoryDepth, StartTime) + uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) + + StartTime = time.mktime( + time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime( + time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": GetH08_dict['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": GetH08_dict['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + + +def scan_Sentinel1_dir(collectionCode, path): + """ + 获取哨兵1号卫星元数据 + :return: + """ + # file_dir = 'E:/数管' + print('开始扫描Sentinel1数据集') + #collectionCode = 'S1A_IW_GRDH' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collectionCode) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return + + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + Sentinel1_dict = GetSentinel1Data(path, xmlPath, ThumbnailPath) + + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(Sentinel1_dict['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(Sentinel1_dict['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = Sentinel1_dict['CollectionCode'] + DirectoryDepth = Sentinel1_dict['DirectoryDepth'] + StartTime = Sentinel1_dict['ProduceTime'][0:10] + ' ' + Sentinel1_dict['ProduceTime'][11:19] + # uc = upload_client(path, DirectoryDepth, StartTime) + uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) + + StartTime = time.mktime(time.strptime(Sentinel1_dict['StartTime'][0:10] + ' ' + Sentinel1_dict['StartTime'][11:19], + '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime( + time.strptime(Sentinel1_dict['StopTime'][0:10] + ' ' + Sentinel1_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": Sentinel1_dict['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": Sentinel1_dict['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + # print("========"+suffix) + + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + + +def scan_Sentinel2_dir(collectionCode, path): + """ + 获取哨兵2号卫星元数据 + :return: + """ + # file_dir = 'E:/数管' + print('开始扫描Sentinel2数据集') + #collectionCode = 'S2B' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collectionCode) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return + + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + Sentinel2_dict = GetSentinel2Data(path, xmlPath, ThumbnailPath) + + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(Sentinel2_dict['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(Sentinel2_dict['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = Sentinel2_dict['CollectionCode'] + DirectoryDepth = Sentinel2_dict['DirectoryDepth'] + StartTime = Sentinel2_dict['ProduceTime'][0:10] + ' ' + Sentinel2_dict['ProduceTime'][11:19] + # uc = upload_client(path, DirectoryDepth, StartTime) + uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) + + StartTime = time.mktime(time.strptime(Sentinel2_dict['StartTime'][0:10] + ' ' + Sentinel2_dict['StartTime'][11:19], + '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime( + time.strptime(Sentinel2_dict['StopTime'][0:10] + ' ' + Sentinel2_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": Sentinel2_dict['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": Sentinel2_dict['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + + +def scan_Sentinel3OL_dir(collectionCode, path): + """ + 获取哨兵3号卫星元数据 + :return: + """ + # file_dir = 'E:/数管' + print('开始扫描Sentinel3数据集') + #collectionCode = 'Sentinel3_OLCI' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collectionCode) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return + + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + Sentinel3OL_dict = GetSentinel3OLData(path, xmlPath, ThumbnailPath) + print(Sentinel3OL_dict) + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(Sentinel3OL_dict['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(Sentinel3OL_dict['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = Sentinel3OL_dict['CollectionCode'] + DirectoryDepth = Sentinel3OL_dict['DirectoryDepth'] + StartTime = Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19] + # uc = upload_client(path, DirectoryDepth, StartTime) + uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) + + StartTime = time.mktime( + time.strptime(Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19], + '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime(time.strptime(Sentinel3OL_dict['StopTime'][0:10] + ' ' + Sentinel3OL_dict['StopTime'][11:19], + '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": Sentinel3OL_dict['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": Sentinel3OL_dict['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + # print("========"+suffix) + + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + + +def scan_HJ1_dir(collectionCode, path): + """ + 获取环境1号卫星元数据 + :return: + """ + # file_dir = 'E:/数管' + #file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\HJ-1' + print('开始扫描环境1号数据集') + #collectionCode = 'HJ-1' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collectionCode) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return + + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + GetHJ1Data_dict = GetHJ1Data(path, xmlPath, ThumbnailPath) + + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(GetHJ1Data_dict['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(GetHJ1Data_dict['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = GetHJ1Data_dict['CollectionCode'] + DirectoryDepth = GetHJ1Data_dict['DirectoryDepth'] + StartTime = GetHJ1Data_dict['ProductTime'][0:10] + ' ' + GetHJ1Data_dict['ProductTime'][11:19] + # uc = upload_client(path, DirectoryDepth, StartTime) + uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) + + StartTime = time.mktime( + time.strptime(GetHJ1Data_dict['StartTime'][0:10] + ' ' + GetHJ1Data_dict['StartTime'][11:19], + '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime( + time.strptime(GetHJ1Data_dict['EndTime'][0:10] + ' ' + GetHJ1Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": GetHJ1Data_dict['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": GetHJ1Data_dict['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + # print("========"+suffix) + + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + + +def scan_ZY3_dir(collectionCode, path): + """ + 获取资源3号卫星元数据 + :return: + """ + # file_dir = 'E:/数管' + #file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\ZY-3' + print('开始扫描资源3号数据集') + #collectionCode = 'ZY-3' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collectionCode) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return + + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + GetZY3Data_dict = GetZY3Data(path, xmlPath, ThumbnailPath) + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(GetZY3Data_dict['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(GetZY3Data_dict['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = GetZY3Data_dict['CollectionCode'] + DirectoryDepth = GetZY3Data_dict['DirectoryDepth'] + StartTime = GetZY3Data_dict['ProduceTime'][0:10] + ' ' + GetZY3Data_dict['ProduceTime'][11:19] + uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) + + StartTime = time.mktime( + time.strptime(GetZY3Data_dict['StartTime'][0:10] + ' ' + GetZY3Data_dict['StartTime'][11:19], + '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime( + time.strptime(GetZY3Data_dict['EndTime'][0:10] + ' ' + GetZY3Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": GetZY3Data_dict['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": GetZY3Data_dict['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + # print("========"+suffix) + + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + + +def scan_SNPP_dir(collectionCode, path): + """ + 获取资源3号卫星元数据 + :return: + """ + # file_dir = 'E:/数管' + #file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\VIIRS' + print('开始扫描VNP02IMG数据集') + #collectionCode = 'VNP02IMG' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collectionCode) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return + + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + GetSNPPData_dict = GetSNPPData(path, xmlPath, ThumbnailPath) + print(GetSNPPData_dict) + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(GetSNPPData_dict['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(GetSNPPData_dict['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = GetSNPPData_dict['CollectionCode'] + DirectoryDepth = GetSNPPData_dict['DirectoryDepth'] + StartTime = GetSNPPData_dict['ProductionTime'][0:10] + ' ' + GetSNPPData_dict['ProductionTime'][11:19] + # uc = upload_client(path, DirectoryDepth, StartTime, collectionCode) + uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) + + StartTime = time.mktime( + time.strptime(GetSNPPData_dict['StartTime'][0:10] + ' ' + GetSNPPData_dict['StartTime'][11:19], + '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime(time.strptime(GetSNPPData_dict['EndTime'][0:10] + ' ' + GetSNPPData_dict['EndTime'][11:19], + '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": GetSNPPData_dict['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": GetSNPPData_dict['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + # print("========"+suffix) + + + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + + +if __name__ == '__main__': + # file_dir = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_001/sensingdata' + # list_dir(file_dir) + # file_dir = 'E:/数管' + # + scan_VJ102_dir() + # + # scan_VJ103_dir() + + # scan_GF1_PMS2_dir() + # scan_GF3MDJ_dir() + # scan_H08_dir() + # scan_Sentinel1_dir() + # scan_Sentinel2_dir() + # scan_Sentinel3OL_dir() + # scan_HJ1_dir() + # scan_ZY3_dir() + # scan_SNPP_dir()