1051 lines
46 KiB
Python
1051 lines
46 KiB
Python
"""
|
||
Author : XinYi Song
|
||
Time : 2022/2/12 17:13
|
||
Desc:
|
||
"""
|
||
import os
|
||
import time
|
||
|
||
import requests
|
||
|
||
from application.settings import Config
|
||
from common.tools.dms import dms_login, dms_task_record, dms_sensing_data
|
||
from scan_data.GetMetaInfo import GetGFPMSData, GetGF3MDJData, GetH08Data, GetSentinel1Data, GetSentinel2Data, \
|
||
GetSentinel3OLData, GetHJ1Data, GetZY3Data, GetSNPPData
|
||
from scan_data.example import GetJPSSData
|
||
from upload.upload_client import upload_file_client
|
||
from util.http_file_upload import upload_client
|
||
from util.http_util import httpUtil
|
||
from util.remote_sensing_util import gf4_pmi_001
|
||
from util.xml_util import dict_to_xml
|
||
|
||
|
||
def list_dir(file_dir):
|
||
"""
|
||
通过 listdir 得到的是仅当前路径下的文件名，不包括子目录中的文件，如果需要得到所有文件需要递归
|
||
"""
|
||
# 用户登录
|
||
token_s = dms_login()
|
||
# 判断定时任务是否在进行
|
||
task = dms_task_record(token_s)
|
||
# 如果不是空说明正在进行
|
||
if task is not None and len(task) > 0:
|
||
return
|
||
fileNameList = []
|
||
dms_list = dms_sensing_data(token_s)
|
||
for dms in dms_list:
|
||
fileNameList.append(dms['fileName'])
|
||
dir_list = os.listdir(file_dir)
|
||
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
||
d = [y for y in dir_list if y not in fileNameList]
|
||
if d is None or len(d) == 0:
|
||
print('没有多余的遥感数据文件，终止程序')
|
||
return
|
||
file_total_size = ""
|
||
file_total_name = ""
|
||
xmlPath = '' # 解析出的xml文件保存的路径
|
||
ThumbnailPath = '' # 解析出的缩略图保存的路径
|
||
for cur_file in d:
|
||
# 获取文件的绝对路径
|
||
path = os.path.join(file_dir, cur_file)
|
||
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
||
file_name = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.tiff'
|
||
xml_path = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml'
|
||
# 解析遥感数据文件（demo）
|
||
gf4_sensing = gf4_pmi_001(file_name, xml_path)
|
||
file_name = os.path.basename(path).split('.')[0]
|
||
xml_path = 'D:/file/work/pythonyuanma/dms_management/xml/' + file_name + '.xml'
|
||
# 将遥感数据写入xml中，并保存本地文件夹中
|
||
dict_to_xml(gf4_sensing, xml_path)
|
||
|
||
# 配置文件服务器参数
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(xml_path, 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项
|
||
# 上传生成的xml文件到文件服务器
|
||
xml = requests.post(url, data=options, files=files)
|
||
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(gf4_sensing['ThumbnailPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项
|
||
# 上传生成的xml文件到文件服务器
|
||
ThumbnailName = requests.post(url, data=options, files=files)
|
||
|
||
CollectionCode = gf4_sensing['CollectionCode']
|
||
DirectoryDepth = gf4_sensing['DirectoryDepth']
|
||
StartTime = gf4_sensing['StartTime']
|
||
uc = upload_client(path, DirectoryDepth, StartTime)
|
||
|
||
StartTime = time.mktime(time.strptime(gf4_sensing['StartTime'], '%Y-%m-%d %H:%M:%S'))
|
||
EndTime = time.mktime(time.strptime(gf4_sensing['EndTime'], '%Y-%m-%d %H:%M:%S'))
|
||
|
||
# 入库遥感数据
|
||
httpUtil(url=Config.RESING_DATA_URL,
|
||
data={"collectionCode": "GF4_PMI_001", "shootingTimeStartTs": StartTime,
|
||
"shootingTimeEndTs": EndTime,
|
||
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
||
"fileSize": uc['file_size'], "cloudCoverage": gf4_sensing['CloudPercent'],
|
||
"metaInformationFile": xml.json()['path'],
|
||
"thumbnailFile": ThumbnailName.json()['path'],
|
||
"remarks": "", "boundaryGeomStr": gf4_sensing['boundaryGeomStr']},
|
||
token=token_s).post_no_patam_herder()
|
||
|
||
file_total_size = file_total_size + str(uc['file_size'])
|
||
file_total_size = file_total_size + ","
|
||
|
||
file_total_name = file_total_name + uc['fileName']
|
||
file_total_name = file_total_name + ","
|
||
# print("========"+suffix)
|
||
print("{0} : is file!".format(cur_file))
|
||
if os.path.isdir(path):
|
||
print("{0} : is dir!".format(cur_file))
|
||
list_dir(path) # 递归子目录
|
||
if uc['type'] == 'ok':
|
||
continue
|
||
# 添加遥感数据归档任务
|
||
res = httpUtil(url=Config.DATA_TASK_URL,
|
||
data={"clientCode": "client1", "collectionCode": "GF4_PMI_001", "storageFileList": file_total_name,
|
||
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
||
task_code = res.json()['data']
|
||
|
||
# 结束遥感数据归档任务
|
||
header = {"Authorization": token_s}
|
||
res = requests.post(url=Config.DATA_END_TASK_URL,
|
||
|
||
params={"taskCode": task_code}, headers=header).json()
|
||
|
||
|
||
def scan_VJ102_dir(collectionCode, path):
|
||
"""
|
||
解析JPSS-VJ102元数据
|
||
:return:
|
||
"""
|
||
print('开始扫描VJ102IMG数据集')
|
||
#collectionCode = 'VJ102IMG'
|
||
# 用户登录
|
||
token_s = dms_login()
|
||
# 判断定时任务是否在进行
|
||
task = dms_task_record(token_s, collectionCode)
|
||
# 如果不是空说明正在进行
|
||
if task is not None and len(task) > 0:
|
||
return 1
|
||
|
||
file_total_size = ""
|
||
file_total_name = ""
|
||
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
||
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
||
|
||
# 解析遥感数据文件（demo）
|
||
JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath)
|
||
|
||
# 配置文件服务器参数
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(JPSSData_dict['xmlPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
xml = requests.post(url, data=options, files=files)
|
||
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
ThumbnailName = requests.post(url, data=options, files=files)
|
||
|
||
CollectionCode = JPSSData_dict['CollectionCode']
|
||
DirectoryDepth = JPSSData_dict['DirectoryDepth']
|
||
StartTime = JPSSData_dict['StartTime']
|
||
# uc = upload_client(path, DirectoryDepth, StartTime[0:19])
|
||
uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode)
|
||
|
||
StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
||
EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
||
|
||
# 入库遥感数据
|
||
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
||
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
||
"shootingTimeEndTs": EndTime,
|
||
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
||
"fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'],
|
||
"metaInformationFile": xml.json()['path'],
|
||
"thumbnailFile": ThumbnailName.json()['path'],
|
||
"remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']},
|
||
token=token_s).post_no_patam_herder()
|
||
print(res_data.json()['data'])
|
||
|
||
file_total_size = file_total_size + str(uc['file_size'])
|
||
file_total_size = file_total_size + ","
|
||
|
||
file_total_name = file_total_name + uc['fileName']
|
||
file_total_name = file_total_name + ","
|
||
|
||
# 添加遥感数据归档任务
|
||
res = httpUtil(url=Config.DATA_TASK_URL,
|
||
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
||
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
||
task_code = res.json()['data']
|
||
|
||
# 结束遥感数据归档任务
|
||
header = {"Authorization": token_s}
|
||
res = requests.post(url=Config.DATA_END_TASK_URL,
|
||
|
||
params={"taskCode": task_code}, headers=header).json()
|
||
if res_data.json()['status'] == 0:
|
||
return 0
|
||
else:
|
||
return 2
|
||
|
||
|
||
def scan_VJ103_dir(collectionCode, path):
|
||
"""
|
||
解析JPSS-VJ103元数据
|
||
:param file_dir:
|
||
:return:
|
||
"""
|
||
# file_dir = 'E:/数管'
|
||
file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\JPSS'
|
||
print('开始扫描VJ103IMG数据集')
|
||
#collectionCode = 'VJ103IMG'
|
||
# 用户登录
|
||
token_s = dms_login()
|
||
# 判断定时任务是否在进行
|
||
task = dms_task_record(token_s, collectionCode)
|
||
# 如果不是空说明正在进行
|
||
if task is not None and len(task) > 0:
|
||
return 1
|
||
file_total_size = ""
|
||
file_total_name = ""
|
||
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
||
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
||
|
||
# 解析遥感数据文件（demo）
|
||
JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath)
|
||
|
||
# 配置文件服务器参数
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(JPSSData_dict['xmlPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
xml = requests.post(url, data=options, files=files)
|
||
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
ThumbnailName = requests.post(url, data=options, files=files)
|
||
|
||
CollectionCode = JPSSData_dict['CollectionCode']
|
||
DirectoryDepth = JPSSData_dict['DirectoryDepth']
|
||
StartTime = JPSSData_dict['StartTime']
|
||
# uc = upload_client(path, DirectoryDepth, StartTime[0:19])
|
||
uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode)
|
||
|
||
StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
||
EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
||
|
||
# 入库遥感数据
|
||
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
||
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
||
"shootingTimeEndTs": EndTime,
|
||
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
||
"fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'],
|
||
"metaInformationFile": xml.json()['path'],
|
||
"thumbnailFile": ThumbnailName.json()['path'],
|
||
"remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']},
|
||
token=token_s).post_no_patam_herder()
|
||
print(res_data.json()['data'])
|
||
|
||
file_total_size = file_total_size + str(uc['file_size'])
|
||
file_total_size = file_total_size + ","
|
||
|
||
file_total_name = file_total_name + uc['fileName']
|
||
file_total_name = file_total_name + ","
|
||
# print("========"+suffix)
|
||
# 添加遥感数据归档任务
|
||
res = httpUtil(url=Config.DATA_TASK_URL,
|
||
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
||
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
||
task_code = res.json()['data']
|
||
|
||
# 结束遥感数据归档任务
|
||
header = {"Authorization": token_s}
|
||
res = requests.post(url=Config.DATA_END_TASK_URL,
|
||
|
||
params={"taskCode": task_code}, headers=header).json()
|
||
if res_data.json()['status'] == 0:
|
||
return 0
|
||
else:
|
||
return 2
|
||
|
||
|
||
def scan_GF1_PMS2_dir(collectionCode, path):
|
||
"""
|
||
获取高分 PMS卫星元数据
|
||
:return:
|
||
"""
|
||
|
||
print('开始扫描GF1_PMS2_001数据集')
|
||
#collectionCode = 'GF1_PMS2_001'
|
||
# 用户登录
|
||
token_s = dms_login()
|
||
# 判断定时任务是否在进行
|
||
task = dms_task_record(token_s, collectionCode)
|
||
# 如果不是空说明正在进行
|
||
if task is not None and len(task) > 0:
|
||
return 1
|
||
file_total_size = ""
|
||
file_total_name = ""
|
||
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
||
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
||
|
||
# 解析遥感数据文件（demo）
|
||
GFPMS_dict = GetGFPMSData(path, xmlPath, ThumbnailPath)
|
||
|
||
# 配置文件服务器参数
|
||
url = Config.DFS_UPLOAD_URL
|
||
print(GFPMS_dict['xmlPath'])
|
||
files = {'file': open(GFPMS_dict['xmlPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
xml = requests.post(url, data=options, files=files)
|
||
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(GFPMS_dict['ThumbnailPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
ThumbnailName = requests.post(url, data=options, files=files)
|
||
|
||
CollectionCode = GFPMS_dict['CollectionCode']
|
||
DirectoryDepth = GFPMS_dict['DirectoryDepth']
|
||
StartTime = GFPMS_dict['StartTime']
|
||
# uc = upload_client(path, DirectoryDepth, StartTime[0:19])
|
||
uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode)
|
||
|
||
StartTime = time.mktime(time.strptime(GFPMS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
||
EndTime = time.mktime(time.strptime(GFPMS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
||
|
||
# 入库遥感数据
|
||
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
||
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
||
"shootingTimeEndTs": EndTime,
|
||
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
||
"fileSize": uc['file_size'], "cloudCoverage": GFPMS_dict['CloudPercent'],
|
||
"metaInformationFile": xml.json()['path'],
|
||
"thumbnailFile": ThumbnailName.json()['path'],
|
||
"remarks": "", "boundaryGeomStr": GFPMS_dict['boundaryGeomStr']},
|
||
token=token_s).post_no_patam_herder()
|
||
print(res_data.json()['data'])
|
||
|
||
file_total_size = file_total_size + str(uc['file_size'])
|
||
file_total_size = file_total_size + ","
|
||
|
||
file_total_name = file_total_name + uc['fileName']
|
||
file_total_name = file_total_name + ","
|
||
|
||
# 添加遥感数据归档任务
|
||
res = httpUtil(url=Config.DATA_TASK_URL,
|
||
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
||
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
||
task_code = res.json()['data']
|
||
|
||
# 结束遥感数据归档任务
|
||
header = {"Authorization": token_s}
|
||
res = requests.post(url=Config.DATA_END_TASK_URL,
|
||
|
||
params={"taskCode": task_code}, headers=header).json()
|
||
if res_data.json()['status'] == 0:
|
||
return 0
|
||
else:
|
||
return 2
|
||
|
||
|
||
def scan_GF3MDJ_dir(collectionCode, path):
|
||
"""
|
||
获取高分3号MDJ（GF-3 MDJ）卫星元数据
|
||
:return:
|
||
"""
|
||
# file_dir = 'E:/数管'
|
||
print('开始扫描GF3_MDJ_SS数据集')
|
||
#collectionCode = 'GF3_MDJ_SS'
|
||
# 用户登录
|
||
token_s = dms_login()
|
||
# 判断定时任务是否在进行
|
||
task = dms_task_record(token_s, collectionCode)
|
||
# 如果不是空说明正在进行
|
||
if task is not None and len(task) > 0:
|
||
return 1
|
||
file_total_size = ""
|
||
file_total_name = ""
|
||
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
||
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
||
|
||
# 解析遥感数据文件（demo）
|
||
GF3_MDJ_SS_dict = GetGF3MDJData(path, xmlPath, ThumbnailPath)
|
||
|
||
# 配置文件服务器参数
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(GF3_MDJ_SS_dict['xmlPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
xml = requests.post(url, data=options, files=files)
|
||
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(GF3_MDJ_SS_dict['ThumbnailPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
ThumbnailName = requests.post(url, data=options, files=files)
|
||
|
||
CollectionCode = GF3_MDJ_SS_dict['CollectionCode']
|
||
DirectoryDepth = GF3_MDJ_SS_dict['DirectoryDepth']
|
||
StartTime = GF3_MDJ_SS_dict['StartTime']
|
||
# uc = upload_client(path, DirectoryDepth, StartTime[0:19])
|
||
uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode)
|
||
|
||
StartTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
||
EndTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
||
|
||
# 入库遥感数据
|
||
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
||
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
||
"shootingTimeEndTs": EndTime,
|
||
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
||
"fileSize": uc['file_size'], "cloudCoverage": GF3_MDJ_SS_dict['CloudPercent'],
|
||
"metaInformationFile": xml.json()['path'],
|
||
"thumbnailFile": ThumbnailName.json()['path'],
|
||
"remarks": "", "boundaryGeomStr": GF3_MDJ_SS_dict['boundaryGeomStr']},
|
||
token=token_s).post_no_patam_herder()
|
||
print(res_data.json()['data'])
|
||
|
||
file_total_size = file_total_size + str(uc['file_size'])
|
||
file_total_size = file_total_size + ","
|
||
|
||
file_total_name = file_total_name + uc['fileName']
|
||
file_total_name = file_total_name + ","
|
||
# print("========"+suffix)
|
||
|
||
# 添加遥感数据归档任务
|
||
res = httpUtil(url=Config.DATA_TASK_URL,
|
||
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
||
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
||
task_code = res.json()['data']
|
||
|
||
# 结束遥感数据归档任务
|
||
header = {"Authorization": token_s}
|
||
res = requests.post(url=Config.DATA_END_TASK_URL,
|
||
|
||
params={"taskCode": task_code}, headers=header).json()
|
||
if res_data.json()['status'] == 0:
|
||
return 0
|
||
else:
|
||
return 2
|
||
|
||
|
||
def scan_H08_dir(collectionCode, path):
|
||
"""
|
||
获取葵花8卫星元数据
|
||
:return:
|
||
"""
|
||
print('开始扫描H08数据集')
|
||
#collectionCode = 'NC_H08'
|
||
# 用户登录
|
||
token_s = dms_login()
|
||
# 判断定时任务是否在进行
|
||
task = dms_task_record(token_s, collectionCode)
|
||
# 如果不是空说明正在进行
|
||
if task is not None and len(task) > 0:
|
||
return 1
|
||
|
||
file_total_size = ""
|
||
file_total_name = ""
|
||
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
||
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
||
|
||
# 解析遥感数据文件（demo）
|
||
GetH08_dict = GetH08Data(path, xmlPath, ThumbnailPath)
|
||
|
||
# 配置文件服务器参数
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(GetH08_dict['xmlPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
xml = requests.post(url, data=options, files=files)
|
||
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(GetH08_dict['ThumbnailPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
ThumbnailName = requests.post(url, data=options, files=files)
|
||
|
||
CollectionCode = GetH08_dict['CollectionCode']
|
||
DirectoryDepth = GetH08_dict['DirectoryDepth']
|
||
StartTime = GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19]
|
||
# uc = upload_client(path, DirectoryDepth, StartTime)
|
||
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
|
||
|
||
StartTime = time.mktime(
|
||
time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
||
EndTime = time.mktime(
|
||
time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
||
|
||
# 入库遥感数据
|
||
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
||
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
||
"shootingTimeEndTs": EndTime,
|
||
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
||
"fileSize": uc['file_size'], "cloudCoverage": GetH08_dict['CloudPercent'],
|
||
"metaInformationFile": xml.json()['path'],
|
||
"thumbnailFile": ThumbnailName.json()['path'],
|
||
"remarks": "", "boundaryGeomStr": GetH08_dict['boundaryGeomStr']},
|
||
token=token_s).post_no_patam_herder()
|
||
print(res_data.json()['data'])
|
||
|
||
file_total_size = file_total_size + str(uc['file_size'])
|
||
file_total_size = file_total_size + ","
|
||
|
||
file_total_name = file_total_name + uc['fileName']
|
||
file_total_name = file_total_name + ","
|
||
|
||
# 添加遥感数据归档任务
|
||
res = httpUtil(url=Config.DATA_TASK_URL,
|
||
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
||
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
||
task_code = res.json()['data']
|
||
|
||
# 结束遥感数据归档任务
|
||
header = {"Authorization": token_s}
|
||
res = requests.post(url=Config.DATA_END_TASK_URL,
|
||
|
||
params={"taskCode": task_code}, headers=header).json()
|
||
if res_data.json()['status'] == 0:
|
||
return 0
|
||
else:
|
||
return 2
|
||
|
||
|
||
def scan_Sentinel1_dir(collectionCode, path):
|
||
"""
|
||
获取哨兵1号卫星元数据
|
||
:return:
|
||
"""
|
||
# file_dir = 'E:/数管'
|
||
print('开始扫描Sentinel1数据集')
|
||
#collectionCode = 'S1A_IW_GRDH'
|
||
# 用户登录
|
||
token_s = dms_login()
|
||
# 判断定时任务是否在进行
|
||
task = dms_task_record(token_s, collectionCode)
|
||
# 如果不是空说明正在进行
|
||
if task is not None and len(task) > 0:
|
||
return 1
|
||
|
||
file_total_size = ""
|
||
file_total_name = ""
|
||
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
||
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
||
|
||
# 解析遥感数据文件（demo）
|
||
Sentinel1_dict = GetSentinel1Data(path, xmlPath, ThumbnailPath)
|
||
|
||
# 配置文件服务器参数
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(Sentinel1_dict['xmlPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
xml = requests.post(url, data=options, files=files)
|
||
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(Sentinel1_dict['ThumbnailPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
ThumbnailName = requests.post(url, data=options, files=files)
|
||
|
||
CollectionCode = Sentinel1_dict['CollectionCode']
|
||
DirectoryDepth = Sentinel1_dict['DirectoryDepth']
|
||
StartTime = Sentinel1_dict['ProduceTime'][0:10] + ' ' + Sentinel1_dict['ProduceTime'][11:19]
|
||
# uc = upload_client(path, DirectoryDepth, StartTime)
|
||
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
|
||
|
||
StartTime = time.mktime(time.strptime(Sentinel1_dict['StartTime'][0:10] + ' ' + Sentinel1_dict['StartTime'][11:19],
|
||
'%Y-%m-%d %H:%M:%S'))
|
||
EndTime = time.mktime(
|
||
time.strptime(Sentinel1_dict['StopTime'][0:10] + ' ' + Sentinel1_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
||
|
||
# 入库遥感数据
|
||
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
||
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
||
"shootingTimeEndTs": EndTime,
|
||
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
||
"fileSize": uc['file_size'], "cloudCoverage": Sentinel1_dict['CloudPercent'],
|
||
"metaInformationFile": xml.json()['path'],
|
||
"thumbnailFile": ThumbnailName.json()['path'],
|
||
"remarks": "", "boundaryGeomStr": Sentinel1_dict['boundaryGeomStr']},
|
||
token=token_s).post_no_patam_herder()
|
||
print(res_data.json()['data'])
|
||
|
||
file_total_size = file_total_size + str(uc['file_size'])
|
||
file_total_size = file_total_size + ","
|
||
|
||
file_total_name = file_total_name + uc['fileName']
|
||
file_total_name = file_total_name + ","
|
||
# print("========"+suffix)
|
||
|
||
# 添加遥感数据归档任务
|
||
res = httpUtil(url=Config.DATA_TASK_URL,
|
||
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
||
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
||
task_code = res.json()['data']
|
||
|
||
# 结束遥感数据归档任务
|
||
header = {"Authorization": token_s}
|
||
res = requests.post(url=Config.DATA_END_TASK_URL,
|
||
|
||
params={"taskCode": task_code}, headers=header).json()
|
||
if res_data.json()['status'] == 0:
|
||
return 0
|
||
else:
|
||
return 2
|
||
|
||
|
||
def scan_Sentinel2_dir(collectionCode, path):
|
||
"""
|
||
获取哨兵2号卫星元数据
|
||
:return:
|
||
"""
|
||
# file_dir = 'E:/数管'
|
||
print('开始扫描Sentinel2数据集')
|
||
#collectionCode = 'S2B'
|
||
# 用户登录
|
||
token_s = dms_login()
|
||
# 判断定时任务是否在进行
|
||
task = dms_task_record(token_s, collectionCode)
|
||
# 如果不是空说明正在进行
|
||
if task is not None and len(task) > 0:
|
||
return 1
|
||
|
||
file_total_size = ""
|
||
file_total_name = ""
|
||
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
||
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
||
|
||
# 解析遥感数据文件（demo）
|
||
Sentinel2_dict = GetSentinel2Data(path, xmlPath, ThumbnailPath)
|
||
|
||
# 配置文件服务器参数
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(Sentinel2_dict['xmlPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
xml = requests.post(url, data=options, files=files)
|
||
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(Sentinel2_dict['ThumbnailPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
ThumbnailName = requests.post(url, data=options, files=files)
|
||
|
||
CollectionCode = Sentinel2_dict['CollectionCode']
|
||
DirectoryDepth = Sentinel2_dict['DirectoryDepth']
|
||
StartTime = Sentinel2_dict['ProduceTime'][0:10] + ' ' + Sentinel2_dict['ProduceTime'][11:19]
|
||
# uc = upload_client(path, DirectoryDepth, StartTime)
|
||
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
|
||
|
||
StartTime = time.mktime(time.strptime(Sentinel2_dict['StartTime'][0:10] + ' ' + Sentinel2_dict['StartTime'][11:19],
|
||
'%Y-%m-%d %H:%M:%S'))
|
||
EndTime = time.mktime(
|
||
time.strptime(Sentinel2_dict['StopTime'][0:10] + ' ' + Sentinel2_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
||
|
||
# 入库遥感数据
|
||
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
||
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
||
"shootingTimeEndTs": EndTime,
|
||
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
||
"fileSize": uc['file_size'], "cloudCoverage": Sentinel2_dict['CloudPercent'],
|
||
"metaInformationFile": xml.json()['path'],
|
||
"thumbnailFile": ThumbnailName.json()['path'],
|
||
"remarks": "", "boundaryGeomStr": Sentinel2_dict['boundaryGeomStr']},
|
||
token=token_s).post_no_patam_herder()
|
||
print(res_data.json()['data'])
|
||
|
||
file_total_size = file_total_size + str(uc['file_size'])
|
||
file_total_size = file_total_size + ","
|
||
|
||
file_total_name = file_total_name + uc['fileName']
|
||
file_total_name = file_total_name + ","
|
||
|
||
# 添加遥感数据归档任务
|
||
res = httpUtil(url=Config.DATA_TASK_URL,
|
||
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
||
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
||
task_code = res.json()['data']
|
||
|
||
# 结束遥感数据归档任务
|
||
header = {"Authorization": token_s}
|
||
res = requests.post(url=Config.DATA_END_TASK_URL,
|
||
|
||
params={"taskCode": task_code}, headers=header).json()
|
||
if res_data.json()['status'] == 0:
|
||
return 0
|
||
else:
|
||
return 2
|
||
|
||
|
||
def scan_Sentinel3OL_dir(collectionCode, path):
|
||
"""
|
||
获取哨兵3号卫星元数据
|
||
:return:
|
||
"""
|
||
# file_dir = 'E:/数管'
|
||
print('开始扫描Sentinel3数据集')
|
||
#collectionCode = 'Sentinel3_OLCI'
|
||
# 用户登录
|
||
token_s = dms_login()
|
||
# 判断定时任务是否在进行
|
||
task = dms_task_record(token_s, collectionCode)
|
||
# 如果不是空说明正在进行
|
||
if task is not None and len(task) > 0:
|
||
return 1
|
||
|
||
file_total_size = ""
|
||
file_total_name = ""
|
||
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
||
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
||
|
||
# 解析遥感数据文件（demo）
|
||
Sentinel3OL_dict = GetSentinel3OLData(path, xmlPath, ThumbnailPath)
|
||
print(Sentinel3OL_dict)
|
||
# 配置文件服务器参数
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(Sentinel3OL_dict['xmlPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
xml = requests.post(url, data=options, files=files)
|
||
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(Sentinel3OL_dict['ThumbnailPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
ThumbnailName = requests.post(url, data=options, files=files)
|
||
|
||
CollectionCode = Sentinel3OL_dict['CollectionCode']
|
||
DirectoryDepth = Sentinel3OL_dict['DirectoryDepth']
|
||
StartTime = Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19]
|
||
# uc = upload_client(path, DirectoryDepth, StartTime)
|
||
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
|
||
|
||
StartTime = time.mktime(
|
||
time.strptime(Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19],
|
||
'%Y-%m-%d %H:%M:%S'))
|
||
EndTime = time.mktime(time.strptime(Sentinel3OL_dict['StopTime'][0:10] + ' ' + Sentinel3OL_dict['StopTime'][11:19],
|
||
'%Y-%m-%d %H:%M:%S'))
|
||
|
||
# 入库遥感数据
|
||
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
||
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
||
"shootingTimeEndTs": EndTime,
|
||
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
||
"fileSize": uc['file_size'], "cloudCoverage": Sentinel3OL_dict['CloudPercent'],
|
||
"metaInformationFile": xml.json()['path'],
|
||
"thumbnailFile": ThumbnailName.json()['path'],
|
||
"remarks": "", "boundaryGeomStr": Sentinel3OL_dict['boundaryGeomStr']},
|
||
token=token_s).post_no_patam_herder()
|
||
print(res_data.json()['data'])
|
||
|
||
file_total_size = file_total_size + str(uc['file_size'])
|
||
file_total_size = file_total_size + ","
|
||
|
||
file_total_name = file_total_name + uc['fileName']
|
||
file_total_name = file_total_name + ","
|
||
# print("========"+suffix)
|
||
|
||
# 添加遥感数据归档任务
|
||
res = httpUtil(url=Config.DATA_TASK_URL,
|
||
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
||
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
||
task_code = res.json()['data']
|
||
|
||
# 结束遥感数据归档任务
|
||
header = {"Authorization": token_s}
|
||
res = requests.post(url=Config.DATA_END_TASK_URL,
|
||
|
||
params={"taskCode": task_code}, headers=header).json()
|
||
if res_data.json()['status'] == 0:
|
||
return 0
|
||
else:
|
||
return 2
|
||
|
||
|
||
def scan_HJ1_dir(collectionCode, path):
|
||
"""
|
||
获取环境1号卫星元数据
|
||
:return:
|
||
"""
|
||
# file_dir = 'E:/数管'
|
||
#file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\HJ-1'
|
||
print('开始扫描环境1号数据集')
|
||
#collectionCode = 'HJ-1'
|
||
# 用户登录
|
||
token_s = dms_login()
|
||
# 判断定时任务是否在进行
|
||
task = dms_task_record(token_s, collectionCode)
|
||
# 如果不是空说明正在进行
|
||
if task is not None and len(task) > 0:
|
||
return 1
|
||
|
||
file_total_size = ""
|
||
file_total_name = ""
|
||
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
||
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
||
|
||
# 解析遥感数据文件（demo）
|
||
GetHJ1Data_dict = GetHJ1Data(path, xmlPath, ThumbnailPath)
|
||
|
||
# 配置文件服务器参数
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(GetHJ1Data_dict['xmlPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
xml = requests.post(url, data=options, files=files)
|
||
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(GetHJ1Data_dict['ThumbnailPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
ThumbnailName = requests.post(url, data=options, files=files)
|
||
|
||
CollectionCode = GetHJ1Data_dict['CollectionCode']
|
||
DirectoryDepth = GetHJ1Data_dict['DirectoryDepth']
|
||
StartTime = GetHJ1Data_dict['ProductTime'][0:10] + ' ' + GetHJ1Data_dict['ProductTime'][11:19]
|
||
# uc = upload_client(path, DirectoryDepth, StartTime)
|
||
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
|
||
|
||
StartTime = time.mktime(
|
||
time.strptime(GetHJ1Data_dict['StartTime'][0:10] + ' ' + GetHJ1Data_dict['StartTime'][11:19],
|
||
'%Y-%m-%d %H:%M:%S'))
|
||
EndTime = time.mktime(
|
||
time.strptime(GetHJ1Data_dict['EndTime'][0:10] + ' ' + GetHJ1Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
||
|
||
# 入库遥感数据
|
||
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
||
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
||
"shootingTimeEndTs": EndTime,
|
||
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
||
"fileSize": uc['file_size'], "cloudCoverage": GetHJ1Data_dict['CloudPercent'],
|
||
"metaInformationFile": xml.json()['path'],
|
||
"thumbnailFile": ThumbnailName.json()['path'],
|
||
"remarks": "", "boundaryGeomStr": GetHJ1Data_dict['boundaryGeomStr']},
|
||
token=token_s).post_no_patam_herder()
|
||
print(res_data.json()['data'])
|
||
|
||
file_total_size = file_total_size + str(uc['file_size'])
|
||
file_total_size = file_total_size + ","
|
||
|
||
file_total_name = file_total_name + uc['fileName']
|
||
file_total_name = file_total_name + ","
|
||
# print("========"+suffix)
|
||
|
||
# 添加遥感数据归档任务
|
||
res = httpUtil(url=Config.DATA_TASK_URL,
|
||
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
||
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
||
task_code = res.json()['data']
|
||
|
||
# 结束遥感数据归档任务
|
||
header = {"Authorization": token_s}
|
||
res = requests.post(url=Config.DATA_END_TASK_URL,
|
||
|
||
params={"taskCode": task_code}, headers=header).json()
|
||
if res_data.json()['status'] == 0:
|
||
return 0
|
||
else:
|
||
return 2
|
||
|
||
|
||
def scan_ZY3_dir(collectionCode, path):
|
||
"""
|
||
获取资源3号卫星元数据
|
||
:return:
|
||
"""
|
||
# file_dir = 'E:/数管'
|
||
#file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\ZY-3'
|
||
print('开始扫描资源3号数据集')
|
||
#collectionCode = 'ZY-3'
|
||
# 用户登录
|
||
token_s = dms_login()
|
||
# 判断定时任务是否在进行
|
||
task = dms_task_record(token_s, collectionCode)
|
||
# 如果不是空说明正在进行
|
||
if task is not None and len(task) > 0:
|
||
return 1
|
||
|
||
file_total_size = ""
|
||
file_total_name = ""
|
||
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
||
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
||
|
||
# 解析遥感数据文件（demo）
|
||
GetZY3Data_dict = GetZY3Data(path, xmlPath, ThumbnailPath)
|
||
# 配置文件服务器参数
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(GetZY3Data_dict['xmlPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
xml = requests.post(url, data=options, files=files)
|
||
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(GetZY3Data_dict['ThumbnailPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
ThumbnailName = requests.post(url, data=options, files=files)
|
||
|
||
CollectionCode = GetZY3Data_dict['CollectionCode']
|
||
DirectoryDepth = GetZY3Data_dict['DirectoryDepth']
|
||
StartTime = GetZY3Data_dict['ProduceTime'][0:10] + ' ' + GetZY3Data_dict['ProduceTime'][11:19]
|
||
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
|
||
|
||
StartTime = time.mktime(
|
||
time.strptime(GetZY3Data_dict['StartTime'][0:10] + ' ' + GetZY3Data_dict['StartTime'][11:19],
|
||
'%Y-%m-%d %H:%M:%S'))
|
||
EndTime = time.mktime(
|
||
time.strptime(GetZY3Data_dict['EndTime'][0:10] + ' ' + GetZY3Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
||
|
||
# 入库遥感数据
|
||
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
||
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
||
"shootingTimeEndTs": EndTime,
|
||
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
||
"fileSize": uc['file_size'], "cloudCoverage": GetZY3Data_dict['CloudPercent'],
|
||
"metaInformationFile": xml.json()['path'],
|
||
"thumbnailFile": ThumbnailName.json()['path'],
|
||
"remarks": "", "boundaryGeomStr": GetZY3Data_dict['boundaryGeomStr']},
|
||
token=token_s).post_no_patam_herder()
|
||
print(res_data.json()['data'])
|
||
|
||
file_total_size = file_total_size + str(uc['file_size'])
|
||
file_total_size = file_total_size + ","
|
||
|
||
file_total_name = file_total_name + uc['fileName']
|
||
file_total_name = file_total_name + ","
|
||
# print("========"+suffix)
|
||
|
||
# 添加遥感数据归档任务
|
||
res = httpUtil(url=Config.DATA_TASK_URL,
|
||
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
||
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
||
task_code = res.json()['data']
|
||
|
||
# 结束遥感数据归档任务
|
||
header = {"Authorization": token_s}
|
||
res = requests.post(url=Config.DATA_END_TASK_URL,
|
||
|
||
params={"taskCode": task_code}, headers=header).json()
|
||
if res_data.json()['status'] == 0:
|
||
return 0
|
||
else:
|
||
return 2
|
||
|
||
|
||
def scan_SNPP_dir(collectionCode, path):
|
||
"""
|
||
获取资源3号卫星元数据
|
||
:return:
|
||
"""
|
||
# file_dir = 'E:/数管'
|
||
#file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\VIIRS'
|
||
print('开始扫描VNP02IMG数据集')
|
||
#collectionCode = 'VNP02IMG'
|
||
# 用户登录
|
||
token_s = dms_login()
|
||
# 判断定时任务是否在进行
|
||
task = dms_task_record(token_s, collectionCode)
|
||
# 如果不是空说明正在进行
|
||
if task is not None and len(task) > 0:
|
||
return 1
|
||
|
||
file_total_size = ""
|
||
file_total_name = ""
|
||
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
||
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
||
|
||
# 解析遥感数据文件（demo）
|
||
GetSNPPData_dict = GetSNPPData(path, xmlPath, ThumbnailPath)
|
||
print(GetSNPPData_dict)
|
||
# 配置文件服务器参数
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(GetSNPPData_dict['xmlPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
xml = requests.post(url, data=options, files=files)
|
||
|
||
url = Config.DFS_UPLOAD_URL
|
||
files = {'file': open(GetSNPPData_dict['ThumbnailPath'], 'rb')}
|
||
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
||
# 上传生成的xml文件到文件服务器
|
||
ThumbnailName = requests.post(url, data=options, files=files)
|
||
|
||
CollectionCode = GetSNPPData_dict['CollectionCode']
|
||
DirectoryDepth = GetSNPPData_dict['DirectoryDepth']
|
||
StartTime = GetSNPPData_dict['ProductionTime'][0:10] + ' ' + GetSNPPData_dict['ProductionTime'][11:19]
|
||
# uc = upload_client(path, DirectoryDepth, StartTime, collectionCode)
|
||
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
|
||
|
||
StartTime = time.mktime(
|
||
time.strptime(GetSNPPData_dict['StartTime'][0:10] + ' ' + GetSNPPData_dict['StartTime'][11:19],
|
||
'%Y-%m-%d %H:%M:%S'))
|
||
EndTime = time.mktime(time.strptime(GetSNPPData_dict['EndTime'][0:10] + ' ' + GetSNPPData_dict['EndTime'][11:19],
|
||
'%Y-%m-%d %H:%M:%S'))
|
||
|
||
# 入库遥感数据
|
||
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
||
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
||
"shootingTimeEndTs": EndTime,
|
||
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
||
"fileSize": uc['file_size'], "cloudCoverage": GetSNPPData_dict['CloudPercent'],
|
||
"metaInformationFile": xml.json()['path'],
|
||
"thumbnailFile": ThumbnailName.json()['path'],
|
||
"remarks": "", "boundaryGeomStr": GetSNPPData_dict['boundaryGeomStr']},
|
||
token=token_s).post_no_patam_herder()
|
||
print(res_data.json()['data'])
|
||
|
||
file_total_size = file_total_size + str(uc['file_size'])
|
||
file_total_size = file_total_size + ","
|
||
|
||
file_total_name = file_total_name + uc['fileName']
|
||
file_total_name = file_total_name + ","
|
||
# print("========"+suffix)
|
||
|
||
|
||
# 添加遥感数据归档任务
|
||
res = httpUtil(url=Config.DATA_TASK_URL,
|
||
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
||
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
||
task_code = res.json()['data']
|
||
|
||
# 结束遥感数据归档任务
|
||
header = {"Authorization": token_s}
|
||
res = requests.post(url=Config.DATA_END_TASK_URL,
|
||
|
||
params={"taskCode": task_code}, headers=header).json()
|
||
if res_data.json()['status'] == 0:
|
||
return 0
|
||
else:
|
||
return 2
|
||
|
||
|
||
if __name__ == '__main__':
|
||
# file_dir = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_001/sensingdata'
|
||
# list_dir(file_dir)
|
||
# file_dir = 'E:/数管'
|
||
#
|
||
scan_VJ102_dir()
|
||
#
|
||
# scan_VJ103_dir()
|
||
|
||
# scan_GF1_PMS2_dir()
|
||
# scan_GF3MDJ_dir()
|
||
# scan_H08_dir()
|
||
# scan_Sentinel1_dir()
|
||
# scan_Sentinel2_dir()
|
||
# scan_Sentinel3OL_dir()
|
||
# scan_HJ1_dir()
|
||
# scan_ZY3_dir()
|
||
# scan_SNPP_dir()
|