likaining/dms-client
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
c5481ead05
dms-client/util/scan_file_util.py
XinYi Song c5481ead05 1、搭建flask框架。 
2、整合JPSS，葵花8，GF3，哨兵1，哨兵2，哨兵3，资源2号，环境1号，SNPP等遥感数据解析算法。
3、flask中添加扫描各个卫星扫描任务，定时扫描，数据入库
2021-12-01 14:27:49 +08:00

1195 lines
58 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""
Author : XinYi Song
Time : 2021/11/3 9:00
Desc:
"""
import os
import time
import requests
from application.settings import Config
from common.tools.dms import dms_login, dms_task_record, dms_sensing_data
from scan_data.GetMetaInfo import GetGFPMSData, GetGF3MDJData, GetH08Data, GetSentinel1Data, GetSentinel2Data, \
GetSentinel3OLData, GetHJ1Data, GetZY3Data, GetSNPPData
from scan_data.example import GetJPSSData
from util.http_file_upload import upload_client
from util.http_util import httpUtil
from util.remote_sensing_util import gf4_pmi_001
from util.xml_util import dict_to_xml
from apscheduler.schedulers.blocking import BlockingScheduler
sched = BlockingScheduler()
def list_dir(file_dir):
"""
通过 listdir 得到的是仅当前路径下的文件名，不包括子目录中的文件，如果需要得到所有文件需要递归
"""
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件，终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = '' # 解析出的xml文件保存的路径
ThumbnailPath = '' # 解析出的缩略图保存的路径
for cur_file in d:
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
file_name = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.tiff'
xml_path = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml'
# 解析遥感数据文件（demo）
gf4_sensing = gf4_pmi_001(file_name, xml_path)
file_name = os.path.basename(path).split('.')[0]
xml_path = 'D:/file/work/pythonyuanma/dms_management/xml/' + file_name + '.xml'
# 将遥感数据写入xml中，并保存本地文件夹中
dict_to_xml(gf4_sensing, xml_path)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(xml_path, 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(gf4_sensing['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = gf4_sensing['CollectionCode']
DirectoryDepth = gf4_sensing['DirectoryDepth']
StartTime = gf4_sensing['StartTime']
uc = upload_client(path, DirectoryDepth, StartTime)
StartTime = time.mktime(time.strptime(gf4_sensing['StartTime'], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(gf4_sensing['EndTime'], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": "GF4_PMI_001", "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": gf4_sensing['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": gf4_sensing['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": "GF4_PMI_001", "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
def scan_VJ102_dir():
"""
解析JPSS-VJ102元数据
:return:
"""
file_dir = 'E:/数管'
print('开始扫描VJ102IMG数据集')
collectionCode = 'VJ102IMG'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s, collectionCode)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件，终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
for cur_file in d:
if 'VJ102IMG' in cur_file and os.path.splitext(cur_file)[1] == '.nc':
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
print(path)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
# 解析遥感数据文件（demo）
JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(JPSSData_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = JPSSData_dict['CollectionCode']
DirectoryDepth = JPSSData_dict['DirectoryDepth']
StartTime = JPSSData_dict['StartTime']
uc = upload_client(path, DirectoryDepth, StartTime[0:19])
StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
def scan_VJ103_dir():
"""
解析JPSS-VJ103元数据
:param file_dir:
:return:
"""
file_dir = 'E:/数管'
print('开始扫描VJ103IMG数据集')
collectionCode = 'VJ103IMG'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s, collectionCode)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件，终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
for cur_file in d:
if 'VJ103IMG' in cur_file and os.path.splitext(cur_file)[1] == '.nc':
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
# 解析遥感数据文件（demo）
JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(JPSSData_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = JPSSData_dict['CollectionCode']
DirectoryDepth = JPSSData_dict['DirectoryDepth']
StartTime = JPSSData_dict['StartTime']
uc = upload_client(path, DirectoryDepth, StartTime[0:19])
StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
def scan_GF1_PMS2_dir():
"""
获取高分 PMS卫星元数据
:return:
"""
file_dir = 'E:/数管'
print('开始扫描GF1_PMS2_001数据集')
collectionCode = 'GF1_PMS2_001'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s, collectionCode)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件，终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
for cur_file in d:
if 'GF1_PMS2' in cur_file[0:8] and os.path.splitext(cur_file)[1] == '.gz':
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
# 解析遥感数据文件（demo）
GFPMS_dict = GetGFPMSData(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
print(GFPMS_dict['xmlPath'])
files = {'file': open(GFPMS_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(GFPMS_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = GFPMS_dict['CollectionCode']
DirectoryDepth = GFPMS_dict['DirectoryDepth']
StartTime = GFPMS_dict['StartTime']
uc = upload_client(path, DirectoryDepth, StartTime[0:19])
StartTime = time.mktime(time.strptime(GFPMS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(GFPMS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": GFPMS_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": GFPMS_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
def scan_GF3MDJ_dir():
"""
获取高分3号MDJ（GF-3 MDJ）卫星元数据
:return:
"""
file_dir = 'E:/数管'
print('开始扫描GF3_MDJ_SS数据集')
collectionCode = 'GF3_MDJ_SS'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s, collectionCode)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件，终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
for cur_file in d:
if 'GF3_MDJ' in cur_file[0:7] and os.path.splitext(cur_file)[1] == '.gz':
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
# 解析遥感数据文件（demo）
GF3_MDJ_SS_dict = GetGF3MDJData(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(GF3_MDJ_SS_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(GF3_MDJ_SS_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = GF3_MDJ_SS_dict['CollectionCode']
DirectoryDepth = GF3_MDJ_SS_dict['DirectoryDepth']
StartTime = GF3_MDJ_SS_dict['StartTime']
uc = upload_client(path, DirectoryDepth, StartTime[0:19])
StartTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": GF3_MDJ_SS_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": GF3_MDJ_SS_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
def scan_H08_dir():
"""
获取高分3号MDJ（GF-3 MDJ）卫星元数据
:return:
"""
file_dir = 'E:/数管'
print('开始扫描H08数据集')
collectionCode = 'NC_H08'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s, collectionCode)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件，终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
for cur_file in d:
if 'NC_H08' in cur_file[0:6] and os.path.splitext(cur_file)[1] == '.nc':
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
# 解析遥感数据文件（demo）
GetH08_dict = GetH08Data(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetH08_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetH08_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = GetH08_dict['CollectionCode']
DirectoryDepth = GetH08_dict['DirectoryDepth']
StartTime = GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19]
uc = upload_client(path, DirectoryDepth, StartTime)
StartTime = time.mktime(time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(GetH08_dict['ProduceTime'][0:10] + ' '+ GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": GetH08_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": GetH08_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
def scan_Sentinel1_dir():
"""
获取哨兵1号卫星元数据
:return:
"""
file_dir = 'E:/数管'
print('开始扫描Sentinel1数据集')
collectionCode = 'S1A_IW_GRDH'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s, collectionCode)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件，终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
for cur_file in d:
if 'S1A_IW_GRDH' in cur_file[0:11] and os.path.splitext(cur_file)[1] == '.zip':
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
# 解析遥感数据文件（demo）
Sentinel1_dict = GetSentinel1Data(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(Sentinel1_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(Sentinel1_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = Sentinel1_dict['CollectionCode']
DirectoryDepth = Sentinel1_dict['DirectoryDepth']
StartTime = Sentinel1_dict['ProduceTime'][0:10] + ' ' + Sentinel1_dict['ProduceTime'][11:19]
uc = upload_client(path, DirectoryDepth, StartTime)
StartTime = time.mktime(time.strptime(Sentinel1_dict['StartTime'][0:10] + ' ' + Sentinel1_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(Sentinel1_dict['StopTime'][0:10] + ' ' + Sentinel1_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": Sentinel1_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": Sentinel1_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
def scan_Sentinel2_dir():
"""
获取哨兵2号卫星元数据
:return:
"""
file_dir = 'E:/数管'
print('开始扫描Sentinel2数据集')
collectionCode = 'S2B'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s, collectionCode)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件，终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
for cur_file in d:
if 'S2B' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.zip':
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
# 解析遥感数据文件（demo）
Sentinel2_dict = GetSentinel2Data(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(Sentinel2_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(Sentinel2_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = Sentinel2_dict['CollectionCode']
DirectoryDepth = Sentinel2_dict['DirectoryDepth']
StartTime = Sentinel2_dict['ProduceTime'][0:10] + ' ' + Sentinel2_dict['ProduceTime'][11:19]
uc = upload_client(path, DirectoryDepth, StartTime)
StartTime = time.mktime(time.strptime(Sentinel2_dict['StartTime'][0:10] + ' ' + Sentinel2_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(Sentinel2_dict['StopTime'][0:10] + ' ' + Sentinel2_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": Sentinel2_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": Sentinel2_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
def scan_Sentinel3OL_dir():
"""
获取哨兵3号卫星元数据
:return:
"""
file_dir = 'E:/数管'
print('开始扫描Sentinel3数据集')
collectionCode = 'Sentinel3_OLCI'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s, collectionCode)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件，终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
for cur_file in d:
if 'S3B' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.zip':
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
print(path)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
# 解析遥感数据文件（demo）
Sentinel3OL_dict = GetSentinel3OLData(path, xmlPath, ThumbnailPath)
print(Sentinel3OL_dict)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(Sentinel3OL_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(Sentinel3OL_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = Sentinel3OL_dict['CollectionCode']
DirectoryDepth = Sentinel3OL_dict['DirectoryDepth']
StartTime = Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19]
uc = upload_client(path, DirectoryDepth, StartTime)
StartTime = time.mktime(time.strptime(Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(Sentinel3OL_dict['StopTime'][0:10] + ' ' + Sentinel3OL_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": Sentinel3OL_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": Sentinel3OL_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
def scan_HJ1_dir():
"""
获取环境1号卫星元数据
:return:
"""
file_dir = 'E:/数管'
print('开始扫描环境1号数据集')
collectionCode = 'HJ-1'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s, collectionCode)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件，终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
for cur_file in d:
if 'HJ1' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.gz':
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
# 解析遥感数据文件（demo）
GetHJ1Data_dict = GetHJ1Data(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetHJ1Data_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetHJ1Data_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = GetHJ1Data_dict['CollectionCode']
DirectoryDepth = GetHJ1Data_dict['DirectoryDepth']
StartTime = GetHJ1Data_dict['ProductTime'][0:10] + ' ' + GetHJ1Data_dict['ProductTime'][11:19]
uc = upload_client(path, DirectoryDepth, StartTime)
StartTime = time.mktime(time.strptime(GetHJ1Data_dict['StartTime'][0:10] + ' ' + GetHJ1Data_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(GetHJ1Data_dict['EndTime'][0:10] + ' ' + GetHJ1Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": GetHJ1Data_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": GetHJ1Data_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
def scan_ZY3_dir():
"""
获取资源3号卫星元数据
:return:
"""
file_dir = 'E:/数管'
print('开始扫描资源3号数据集')
collectionCode = 'ZY-3'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s, collectionCode)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件，终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
for cur_file in d:
if 'ZY3' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.gz':
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
# 解析遥感数据文件（demo）
GetZY3Data_dict = GetZY3Data(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetZY3Data_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetZY3Data_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = GetZY3Data_dict['CollectionCode']
DirectoryDepth = GetZY3Data_dict['DirectoryDepth']
StartTime = GetZY3Data_dict['ProduceTime'][0:10] + ' ' + GetZY3Data_dict['ProduceTime'][11:19]
uc = upload_client(path, DirectoryDepth, StartTime)
StartTime = time.mktime(time.strptime(GetZY3Data_dict['StartTime'][0:10] + ' ' + GetZY3Data_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(GetZY3Data_dict['EndTime'][0:10] + ' ' + GetZY3Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": GetZY3Data_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": GetZY3Data_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
def scan_SNPP_dir():
"""
获取资源3号卫星元数据
:return:
"""
file_dir = 'E:/数管'
print('开始扫描VNP02IMG数据集')
collectionCode = 'VNP02IMG'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s, collectionCode)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件，终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
for cur_file in d:
if 'VNP02IMG' in cur_file[0:8] and os.path.splitext(cur_file)[1] == '.nc':
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
# 解析遥感数据文件（demo）
GetSNPPData_dict = GetSNPPData(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetSNPPData_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetSNPPData_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = GetSNPPData_dict['CollectionCode']
DirectoryDepth = GetSNPPData_dict['DirectoryDepth']
StartTime = GetSNPPData_dict['ProductionTime'][0:10] + ' ' + GetSNPPData_dict['ProductionTime'][11:19]
uc = upload_client(path, DirectoryDepth, StartTime)
StartTime = time.mktime(time.strptime(GetSNPPData_dict['StartTime'][0:10] + ' ' + GetSNPPData_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(GetSNPPData_dict['EndTime'][0:10] + ' ' + GetSNPPData_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": GetSNPPData_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": GetSNPPData_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
if __name__ == '__main__':
# file_dir = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_001/sensingdata'
# list_dir(file_dir)
# file_dir = 'E:/数管'
#
# scan_VJ102_dir()
#
# scan_VJ103_dir()
# scan_GF1_PMS2_dir()
# scan_GF3MDJ_dir()
# scan_H08_dir()
# scan_Sentinel1_dir()
# scan_Sentinel2_dir()
# scan_Sentinel3OL_dir()
# scan_HJ1_dir()
# scan_ZY3_dir()
scan_SNPP_dir()
Reference in New Issue View Git Blame Copy Permalink