diff --git a/__pycache__/app.cpython-39.pyc b/__pycache__/app.cpython-39.pyc deleted file mode 100644 index dc204d4..0000000 Binary files a/__pycache__/app.cpython-39.pyc and /dev/null differ diff --git a/api/hello.py b/api/hello.py new file mode 100644 index 0000000..29aa1b0 --- /dev/null +++ b/api/hello.py @@ -0,0 +1,7 @@ + + +api = RedPrint('hello') + +@api.route('/hello', methods=['GET']) +def index(): + return '

Hello , World

' \ No newline at end of file diff --git a/app.py b/app.py deleted file mode 100644 index 87a9632..0000000 --- a/app.py +++ /dev/null @@ -1,16 +0,0 @@ -from flask import Flask - -from application import init_app -from common.config.factory import create_app - -app = init_app('dev') -app = create_app() - - -@app.route('/') -def hello_world(): - return 'Hello World!' - - -if __name__ == '__main__': - app.run() diff --git a/application/__init__.py b/application/__init__.py index 0742144..89b038f 100644 --- a/application/__init__.py +++ b/application/__init__.py @@ -1,15 +1,10 @@ # 项目初始化文件 -import logging -from logging.handlers import RotatingFileHandler from flask import Flask -from redis import StrictRedis -from flask_wtf.csrf import CSRFProtect from flask_session import Session -from flask_sqlalchemy import SQLAlchemy +from flask_wtf.csrf import CSRFProtect from application.settings.dev import DevelopmentConfig from application.settings.prop import ProductionConfig -from common.config.factory import create_app config = { 'dev': DevelopmentConfig, diff --git a/common/config/factory.py b/common/config/factory.py index 69154c3..d0d0ea2 100644 --- a/common/config/factory.py +++ b/common/config/factory.py @@ -3,14 +3,22 @@ Author : XinYi Song Time : 2021/11/23 9:47 Desc: """ -from flask import Flask +from flask import Flask, Blueprint from flask_apscheduler import APScheduler +from flask_cors import CORS scheduler = APScheduler() +def init_blueprint_v1(): + bp_v1 = Blueprint('/', __name__) + # user.api.register(bp_v1) + return bp_v1 + + def create_app(): app = Flask(__name__) + app.register_blueprint(init_blueprint_v1(), url_prefix='/') # 配置任务，不然无法启动任务 app.config.update( { @@ -80,6 +88,7 @@ def create_app(): ] } ) + CORS(app) scheduler.init_app(app) scheduler.start() return app diff --git a/scan_data/GetMetaInfo.py b/data_meta/GetMetaInfo.py similarity index 75% rename from scan_data/GetMetaInfo.py rename to data_meta/GetMetaInfo.py index 1151087..96c3732 100644 --- a/scan_data/GetMetaInfo.py +++ b/data_meta/GetMetaInfo.py @@ -1,16 +1,16 @@ +import io from xml.dom import minidom from osgeo import gdal -from osgeo import ogr -from osgeo import gdalconst import h5py from PIL import Image import numpy as np -import tarfile -import zipfile import re import os -import io import sys +import time +import datetime +import tarfile +import zipfile def exe_path(): @@ -27,6 +27,14 @@ def exe_path(): os.environ['PROJ_LIB'] = exe_path() + "/PROJ" +def GetTimestamp(date) : + # 转换成时间数组 + timeArray = time.strptime(date, "%Y-%m-%d %H:%M:%S") + # 转换成时间戳 + timestamp = time.mktime(timeArray) + + return timestamp + def uint16to8(bands, lower_percent=0.001, higher_percent=99.999): """ @@ -903,7 +911,7 @@ def GetSentinel2Data(in_file, xml_path, thumbnail_path): try: in_path, basename = os.path.split(in_file) with zipfile.ZipFile(in_file, 'r', zipfile.ZIP_DEFLATED) as zip_file: - extensions = ('_B02_60m.jp2', '_B03_60m.jp2', '_B04_60m.jp2', '.SAFE/MTD_MSIL2A.xml') + extensions = ('_B02.jp2', '_B03.jp2', '_B04.jp2', '.SAFE/MTD_MSIL1C') file_list = [file for file in zip_file.namelist() if file.endswith(extensions)] file_list.sort() @@ -957,7 +965,7 @@ def GetSentinel2Data(in_file, xml_path, thumbnail_path): del img # 解压多光谱XML文件 - if file_list[3].endswith('.SAFE/MTD_MSIL2A.xml'): + if file_list[3].endswith('.SAFE/MTD_MSIL1C'): # 生成XML文件 xmlFileName = os.path.splitext(basename)[0] + ".xml" xmlPath = os.path.join(xml_path, xmlFileName) @@ -1489,66 +1497,529 @@ def GetZY3Data(in_file, xml_path, thumbnail_path): return {"code": -1, "msg": str(e)} +def GetLandsatData(in_file, thumbnail_path, txt_path) : + try : + in_path, basename = os.path.split(in_file) + with tarfile.open(in_file, mode='r') as tar_file : + extensions = ('_B2.TIF', '_B3.TIF', '_B4.TIF', '_MTL.txt') + file_list = [file for file in tar_file.getnames() if file.endswith(extensions)] + file_list.sort() + + # 生成缩略图 + ThumbnailName = os.path.splitext(basename)[0] + "_thumb.jpg" + ThumbnailPath = os.path.join(thumbnail_path, ThumbnailName) + + rgb_list = [] + for file in file_list[:3] : + path = '/vsitar/%s/%s' % (in_file, file) + sub_dataset = gdal.Open(path) + sub_array = sub_dataset.ReadAsArray() + rgb_list.append(sub_array) + img_data = np.array([rgb_list[2], rgb_list[1], rgb_list[0]]) + img_data = uint16to8(img_data) + + # Array转Image + img_data2 = np.transpose(img_data, (1, 2, 0)) + img_data2 = img_data2[:, :, : :-1] + img = Image.fromarray(img_data2) + # 压缩图片大小 + if img_data.shape[1] > img_data.shape[2] : + width = 512 + height = int(width / img_data.shape[1] * img_data.shape[2]) + else : + height = 512 + width = int(height / img_data.shape[1] * img_data.shape[2]) + img.thumbnail((width, height)) + img.save(ThumbnailPath, "PNG") + + # 释放内存 + del rgb_list + del img_data + del img_data2 + del img + + Landsat_dict = {} + + if file_list[3].endswith('_MTL.txt') : + # 读取信息 + TxTPath = os.path.join(txt_path, file_list[3]) + tar_file.extract(file_list[3], txt_path) + + fr = open(TxTPath, 'r') + dic = {} + keys = [] # 用来存储读取的顺序 + for line in fr : + if '=' in line : + v = line.strip().split(' = ') + dic[v[0]] = v[1] + keys.append(v[0]) + fr.close() + + # 读取信息 + cloud_percent = dic['CLOUD_COVER'] + ImageGSD = '30' + ProjectedCoordinates = dic['MAP_PROJECTION'] + + # 产品日期 + FILE_DATE = dic['FILE_DATE'] + FILE_DATE = FILE_DATE.split("Z")[0].replace("T", " ") + ProduceTime = str(GetTimestamp(FILE_DATE)) + + # 经纬度 + TopLeftLatitude = dic['CORNER_UL_LAT_PRODUCT'] # 左上纬度 + TopLeftLongitude = dic['CORNER_UL_LON_PRODUCT'] # 左上经度 + TopRightLatitude = dic['CORNER_UR_LAT_PRODUCT'] # 右上纬度 + TopRightLongitude = dic['CORNER_UR_LON_PRODUCT'] # 右上经度 + BottomRightLatitude = dic['CORNER_LR_LAT_PRODUCT'] # 右下纬度 + BottomRightLongitude = dic['CORNER_LR_LON_PRODUCT'] # 右下经度 + BottomLeftLatitude = dic['CORNER_LL_LAT_PRODUCT'] # 左下纬度 + BottomLeftLongitude = dic['CORNER_LL_LON_PRODUCT'] # 左下经度 + + boundaryGeomStr = f'POLYGON(({TopLeftLongitude} {TopLeftLatitude},' \ + f'{TopRightLongitude} {TopRightLatitude},' \ + f'{BottomRightLongitude} {BottomRightLatitude},' \ + f'{BottomLeftLongitude} {BottomLeftLatitude},' \ + f'{TopLeftLongitude} {TopLeftLatitude}))' + + # 生成字典 + Landsat_dict = {"ProduceTime" : ProduceTime, + "StartTime" : "", + "StopTime" : "", + "CloudPercent" : cloud_percent, + "boundaryGeomStr" : boundaryGeomStr, + # "bands" : "1,2,3,4,5,6,7,8,9,10,11,12", + "ImageGSD" : ImageGSD, + # "ProjectedCoordinates" : ProjectedCoordinates, + # "CollectionCode" : '', + "ThumbnailName" : ThumbnailName, + "ThumbnailPath" : ThumbnailPath, + "xmlPath" : TxTPath, + "xmlFileName" : file_list[3], + "DirectoryDepth" : "month"} + + if not Landsat_dict : + return {"code" : -1, "msg" : "没有满足条件的数据字典..."} + return Landsat_dict + + except Exception as e : + print(str(e)) + return {"code" : -1, "msg" : str(e)} + + +def GetSentinel2Data(in_file, xml_path, thumbnail_path) : + """ + 获取哨兵2卫星元数据 + :param thumbnail_path: + :param in_file: + :return: 元数据字典 + """ + try : + in_path, basename = os.path.split(in_file) + with zipfile.ZipFile(in_file, 'r', zipfile.ZIP_DEFLATED) as zip_file : + extensions = ('_B02.jp2', '_B03.jp2', '_B04.jp2', 'MTD_MSIL1C.xml') + file_list = [file for file in zip_file.namelist() if file.endswith(extensions)] + file_list.sort() + + # 生成缩略图 + ThumbnailName = os.path.splitext(basename)[0] + "_thumb.jpg" + ThumbnailPath = os.path.join(thumbnail_path, ThumbnailName) + + rgb_list = [] + for file in file_list[:3] : + sub_dataset = gdal.Open('/vsizip/%s/%s' % (in_file, file)) + sub_array = sub_dataset.ReadAsArray() + rgb_list.append(sub_array) + img_data = np.array([rgb_list[2], rgb_list[1], rgb_list[0]]) + img_data = uint16to8(img_data) + + # Array转Image + img_data2 = np.transpose(img_data, (1, 2, 0)) + img_data2 = img_data2[:, :, : :-1] + img = Image.fromarray(img_data2) + # 压缩图片大小 + if img_data.shape[1] > img_data.shape[2] : + width = 512 + height = int(width / img_data.shape[1] * img_data.shape[2]) + else : + height = 512 + width = int(height / img_data.shape[1] * img_data.shape[2]) + img.thumbnail((width, height)) + img.save(ThumbnailPath, "PNG") + + # 释放内存 + del rgb_list + del img_data + del img_data2 + del img + + S2_dict = {} + + if file_list[3].endswith('MTD_MSIL1C.xml') : + # 生成xml文件 + xmlFileName = os.path.splitext(basename)[0] + ".xml" + xmlPath = os.path.join(xml_path, xmlFileName) + meta_data = zip_file.read(file_list[3]) + with open(xmlPath, "wb") as fout : + fout.write(meta_data) + + # 读取信息 + meta_content = zip_file.open(file_list[3]) + dom = minidom.parse(meta_content) + cloud_percent = dom.getElementsByTagName('n1:Quality_Indicators_Info')[ + 0].getElementsByTagName('Cloud_Coverage_Assessment')[0].firstChild.data + ImageGSD = '10, 20, 60' + + # 产品日期 + GENERATION_TIME = dom.getElementsByTagName('n1:General_Info')[0].getElementsByTagName('Product_Info')[ + 0].getElementsByTagName('GENERATION_TIME')[0].firstChild.data + GENERATION_TIME = GENERATION_TIME.split(".")[0].replace("T", " ") + ProduceTime = str(GetTimestamp(GENERATION_TIME)) + + PRODUCT_START_TIME = \ + dom.getElementsByTagName('n1:General_Info')[0].getElementsByTagName('Product_Info')[ + 0].getElementsByTagName('PRODUCT_START_TIME')[0].firstChild.data + PRODUCT_START_TIME = PRODUCT_START_TIME.split(".")[0].replace("T", " ") + StartTime = str(GetTimestamp(PRODUCT_START_TIME)) + + PRODUCT_STOP_TIME = dom.getElementsByTagName('n1:General_Info')[0].getElementsByTagName('Product_Info')[ + 0].getElementsByTagName('PRODUCT_STOP_TIME')[0].firstChild.data + PRODUCT_STOP_TIME = PRODUCT_STOP_TIME.split(".")[0].replace("T", " ") + StopTime = str(GetTimestamp(PRODUCT_STOP_TIME)) + + # 经纬度 + lon_lat = dom.getElementsByTagName('n1:Geometric_Info')[0].getElementsByTagName('Product_Footprint')[ + 0].getElementsByTagName('Product_Footprint')[0].getElementsByTagName('Global_Footprint')[ + 0].getElementsByTagName('EXT_POS_LIST')[0].firstChild.data + TopLeftLatitude = re.split(r'\s', lon_lat)[0] # 左上纬度 + TopLeftLongitude = re.split(r'\s', lon_lat)[1] # 左上经度 + TopRightLatitude = re.split(r'\s', lon_lat)[2] # 右上纬度 + TopRightLongitude = re.split(r'\s', lon_lat)[3] # 右上经度 + BottomRightLatitude = re.split(r'\s', lon_lat)[4] # 右下纬度 + BottomRightLongitude = re.split(r'\s', lon_lat)[5] # 右下经度 + BottomLeftLatitude = re.split(r'\s', lon_lat)[6] # 左下纬度 + BottomLeftLongitude = re.split(r'\s', lon_lat)[7] # 左下经度 + + boundaryGeomStr = f'POLYGON(({TopLeftLongitude} {TopLeftLatitude},' \ + f'{TopRightLongitude} {TopRightLatitude},' \ + f'{BottomRightLongitude} {BottomRightLatitude},' \ + f'{BottomLeftLongitude} {BottomLeftLatitude},' \ + f'{TopLeftLongitude} {TopLeftLatitude}))' + + # 生成字典 + S2_dict = {"ProduceTime" : ProduceTime, + "StartTime" : StartTime, + "StopTime" : StopTime, + "CloudPercent" : cloud_percent, + "boundaryGeomStr" : boundaryGeomStr, + "ImageGSD" : ImageGSD, + "ThumbnailName" : ThumbnailName, + "ThumbnailPath" : ThumbnailPath, + "xmlPath" : file_list[3], + "xmlFileName" : xmlFileName, + "DirectoryDepth" : "month"} + + if not S2_dict : + return {"code" : -1, "msg" : "没有满足条件的数据字典..."} + return S2_dict + except Exception as e : + print(str(e)) + return {"code" : -1, "msg" : str(e)} + + +def GetModisData(in_file) : + """ + 获取MODIS卫星元数据 + :param in_file: + :return: 元数据字典 + """ + try : + datasets = gdal.Open(in_file) + # 获取hdf中的元数据 + Metadata = datasets.GetMetadata() + + # 获取信息 + PRODUCTIONDATETIME = Metadata["PRODUCTIONDATETIME"] + PRODUCTIONDATETIME = PRODUCTIONDATETIME.split(".")[0].replace("T", " ") + ProductionTime = str(GetTimestamp(PRODUCTIONDATETIME)) + + RANGEBEGINNINGDATE = Metadata["RANGEBEGINNINGDATE"] + " " + Metadata["RANGEBEGINNINGTIME"] + StartTime = str(GetTimestamp(RANGEBEGINNINGDATE)) + + RANGEENDINGDATE = Metadata["RANGEENDINGDATE"] + " " + Metadata["RANGEENDINGTIME"] + EndTime = str(GetTimestamp(RANGEENDINGDATE)) + + Latitudes = Metadata["GRINGPOINTLATITUDE.1"] # 获取四个角的维度 + LatitudesList = Latitudes.split(", ") # 采用", "进行分割 + Longitude = Metadata["GRINGPOINTLONGITUDE.1"] # 获取四个角的经度 + LongitudeList = Longitude.split(", ") # 采用", "进行分割 + TopLeftLatitude = LatitudesList[0] + TopLeftLongitude = LongitudeList[0] + TopRightLatitude = LatitudesList[1] + TopRightLongitude = LongitudeList[1] + BottomRightLatitude = LatitudesList[2] + BottomRightLongitude = LongitudeList[2] + BottomLeftLatitude = LatitudesList[3] + BottomLeftLongitude = LongitudeList[3] # 获取经纬度 + + boundaryGeomStr = f'POLYGON(({TopLeftLongitude} {TopLeftLatitude},' \ + f'{TopRightLongitude} {TopRightLatitude},' \ + f'{BottomRightLongitude} {BottomRightLatitude},' \ + f'{BottomLeftLongitude} {BottomLeftLatitude},' \ + f'{TopLeftLongitude} {TopLeftLatitude}))' + + DirectoryDepth = Metadata["DAYNIGHTFLAG"] + + modis_dict = {"ProduceTime" : ProductionTime, + "StartTime" : StartTime, + "EndTime" : EndTime, + "ImageGSD" : "", + "CloudPercent" : "0", + 'boundaryGeomStr' : boundaryGeomStr, + "ThumbnailPath" : "", + "ThumbnailName" : "", + "xmlPath" : "", + "xmlFileName" : "", + "DirectoryDepth" : DirectoryDepth + } + + if not modis_dict : + return {"code" : -1, "msg" : "没有满足条件的数据字典..."} + return modis_dict + except Exception as e : + print(str(e)) + return {"code" : -1, "msg" : str(e)} + + +def GetGOCIData(in_file) : + """ + 获取GOCI卫星元数据 + :param in_file: + :return: 元数据字典 + """ + try : + # 读取信息 + gdal.SetConfigOption("GDAL_FILENAME_IS_UTF8", "YES") + in_datasets = gdal.Open(in_file) + meta_data = in_datasets.GetMetadata() + + # 产品日期 + date_created = meta_data['HDFEOS_POINTS_Ephemeris_Scene_center_time'] + date_created = date_created.split(".")[0] + date_created = str(datetime.datetime.strptime(date_created, "%d-%b-%Y %H:%M:%S")) + ProductionTime = str(GetTimestamp(date_created)) + + start_time = meta_data['HDFEOS_POINTS_Ephemeris_Scene_Start_time'] + start_time = start_time.split(".")[0] + start_time = str(datetime.datetime.strptime(start_time, "%d-%b-%Y %H:%M:%S")) + StartTime = str(GetTimestamp(start_time)) + + end_time = meta_data['HDFEOS_POINTS_Ephemeris_Scene_end_time'] + end_time = end_time.split(".")[0] + end_time = str(datetime.datetime.strptime(end_time, "%d-%b-%Y %H:%M:%S")) + EndTime = str(GetTimestamp(end_time)) + + ImageGSD = meta_data['HDFEOS_POINTS_Scene_Header_pixel_spacing'].split(" ")[0] + + # 经纬度 + upper_left_latitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_upper-left_latitude'] + upper_left_longitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_upper-left_longitude'] + upper_right_latitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_upper-right_latitude'] + upper_right_longitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_upper-right_longitude'] + lower_right_latitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_lower-right_latitude'] + lower_right_longitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_lower-right_longitude'] + lower_left_latitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_lower-left_latitude'] + lower_left_longitude = meta_data['HDFEOS_POINTS_Scene_Header_Scene_lower-left_longitude'] + + boundaryGeomStr = f'POLYGON(({upper_left_longitude} {upper_left_latitude},' \ + f'{upper_right_longitude} {upper_right_latitude},' \ + f'{lower_right_longitude} {lower_right_latitude},' \ + f'{lower_left_longitude} {lower_left_latitude},' \ + f'{upper_left_longitude} {upper_left_latitude}))' + + # 构建字典 + GOCI_dict = {"ProduceTime" : ProductionTime, + "StartTime" : StartTime, + "EndTime" : EndTime, + "CloudPercent" : "0", + "boundaryGeomStr" : boundaryGeomStr, + "ImageGSD" : ImageGSD, + "ThumbnailPath" : "", + "ThumbnailName" : "", + "xmlPath" : "", + "xmlFileName" : "", + "DirectoryDepth" : "day"} + + # 判断字典是否为空 + if not GOCI_dict : + return {"code" : -1, "msg" : "没有满足条件的数据字典..."} + return GOCI_dict + except Exception as e : + print(str(e)) + return {"code" : -1, "msg" : str(e)} + + +def GetGK2BData(in_file) : + """ + 获取GK2B_GOCI卫星元数据 + :param in_file: + :return: 元数据字典 + """ + try : + # 读取信息 + gdal.SetConfigOption("GDAL_FILENAME_IS_UTF8", "YES") + in_datasets = gdal.Open(in_file) + meta_data = in_datasets.GetMetadata() + + # 产品日期 + produc_time = meta_data['NC_GLOBAL#slot_acquisition_time'].replace("_", "") + produc_time = produc_time[0 :4] + "-" + produc_time[4 :6] + "-" + produc_time[6 :8] + " " + \ + produc_time[8 :10] + ":" + produc_time[10 :12] + ":" + produc_time[12 :14] + ProductionTime = str(GetTimestamp(produc_time)) + + start_time = meta_data['NC_GLOBAL#observation_start_time'].replace("_", "") + start_time = start_time[0 :4] + "-" + start_time[4 :6] + "-" + start_time[6 :8] + " " + \ + start_time[8 :10] + ":" + start_time[10 :12] + ":" + start_time[12 :14] + StartTime = str(GetTimestamp(start_time)) + + end_time = meta_data['NC_GLOBAL#observation_end_time'].replace("_", "") + end_time = end_time[0 :4] + "-" + end_time[4 :6] + "-" + end_time[6 :8] + " " + \ + end_time[8 :10] + ":" + end_time[10 :12] + ":" + end_time[12 :14] + EndTime = str(GetTimestamp(end_time)) + + # 其他信息 + ImageGSD = meta_data['NC_GLOBAL#geospatial_lat_resolution'].split(" ")[0] + + # 中心经纬度 + productUpperLeftLat = meta_data['NC_GLOBAL#image_upperleft_latitude'] # 左上纬度 + productUpperLeftLong = meta_data['NC_GLOBAL#image_upperleft_longitude'] # 左上经度 + productUpperRightLat = meta_data['NC_GLOBAL#image_upperleft_latitude'] # 右上纬度 + productUpperRightLong = meta_data['NC_GLOBAL#image_lowerright_longitude'] # 右上经度 + productLowerLeftLat = meta_data['NC_GLOBAL#image_lowerright_latitude'] # 左下纬度 + productLowerLeftLong = meta_data['NC_GLOBAL#image_upperleft_longitude'] # 左下经度 + productLowerRightLat = meta_data['NC_GLOBAL#image_lowerright_latitude'] # 右下纬度 + productLowerRightLong = meta_data['NC_GLOBAL#image_lowerright_longitude'] # 右下经度 + + # 边界几何 + boundaryGeomStr = f'POLYGON(({productUpperLeftLong} {productUpperLeftLat},' \ + f'{productUpperRightLong} {productUpperRightLat},' \ + f'{productLowerRightLong} {productLowerRightLat},' \ + f'{productLowerLeftLong} {productLowerLeftLat},' \ + f'{productUpperLeftLong} {productUpperLeftLat}))' + + # 构建字典 + GK2B_dict = {"ProduceTime" : ProductionTime, + "StartTime" : StartTime, + "EndTime" : EndTime, + "CloudPercent" : "0", + "boundaryGeomStr" : boundaryGeomStr, + "ImageGSD" : ImageGSD, + "ThumbnailPath" : "", + "ThumbnailName" : "", + "xmlPath" : "", + "xmlFileName" : "", + "DirectoryDepth" : "day"} + + # 判断字典是否为空 + if not GK2B_dict : + return {"code" : -1, "msg" : "没有满足条件的数据字典..."} + return GK2B_dict + except Exception as e : + print(str(e)) + return {"code" : -1, "msg" : str(e)} + + if __name__ == '__main__': - HJ1FilePath = r"Y:\不同传感器数据\HJ-1\HJ1A-CCD2-450-80-20090501-L20000106616.tar.gz" - JPSSFilePath = r"Y:\不同传感器数据\JPSS\VJ102IMG.A2021159.0542.002.2021159094907.nc" - ZY2FilePath = r"Y:\不同传感器数据\ZY-2\ZY02C_PMS_E115.9_N36.2_20120422_L2C0000391981.tar.gz" - ZY3FilePath = r"Y:\不同传感器数据\ZY-3\ZY3_MUX_E83.3_N43.3_20120405_L2A0000301226.tar.gz" - - S1FilePath = r'Y:\不同传感器数据\SENTINEL-1\S1A_IW_GRDH_1SDV_20210407T095634_20210407T095659_037343_046675_8E66.zip' - S2FilePath = r'Y:\不同传感器数据\SENTINEL-2\S2B_MSIL2A_20210804T024549_N0301_R132_T50SQF_20210804T053331.zip' - GF1PMSPath = r'Y:\不同传感器数据\GF-1\GF1_PMS2_E104.1_N36.6_20210308_L1A0005524847.tar.gz' - H08FilePath = r"Y:\不同传感器数据\葵花8\NC_H08_20210802_2010_R21_FLDK.06001_06001.nc" - SNPPFilePath = r"Y:\不同传感器数据\VIIRS\VNP02IMG.A2021182.0418.001.2021182100800.nc" - - GF3MDJPath = r'Y:\不同传感器数据\GF-3\GF3_MDJ_SS_024986_E120.8_N35.6_20210509_L1A_VHVV_L10005638033.tar.gz' - GF4PMIPath = r'Y:\不同传感器数据\GF-4\GF4_PMI_E119.8_N35.3_20210908_L1A0000417337.tar.gz' - S3OLFilePath = r'Y:\不同传感器数据\SENTINEL-3' \ - r'\S3B_OL_1_EFR____20210910T022645_20210910T022945_20210911T064342_0179_056_374_2340_LN1_O_NT_002.zip' - S3SLFilePath = r'Y:\不同传感器数据\SENTINEL-3' \ - r'\S3A_SL_1_RBT____20210916T020956_20210916T021256_20210917T120953_0179_076_217_2340_LN2_O_NT_004.zip' + # HJ1FilePath = r"Y:\不同传感器数据\HJ-1\HJ1A-CCD2-450-80-20090501-L20000106616.tar.gz" + # JPSSFilePath = r"Y:\不同传感器数据\JPSS\VJ102IMG.A2021159.0542.002.2021159094907.nc" + # ZY2FilePath = r"Y:\不同传感器数据\ZY-2\ZY02C_PMS_E115.9_N36.2_20120422_L2C0000391981.tar.gz" + # ZY3FilePath = r"Y:\不同传感器数据\ZY-3\ZY3_MUX_E83.3_N43.3_20120405_L2A0000301226.tar.gz" + # + # S1FilePath = r'Y:\不同传感器数据\SENTINEL-1\S1A_IW_GRDH_1SDV_20210407T095634_20210407T095659_037343_046675_8E66.zip' + # S2FilePath = r"D:\1Company\Python\RS_data_Dowload\Google_Download_New\data\S2B_MSIL1C_20210113T024049_N0209_R089_T51STB_20210113T041228.zip" + # GF1PMSPath = r'Y:\不同传感器数据\GF-1\GF1_PMS2_E104.1_N36.6_20210308_L1A0005524847.tar.gz' + # H08FilePath = r"Y:\不同传感器数据\葵花8\NC_H08_20210802_2010_R21_FLDK.06001_06001.nc" + # SNPPFilePath = r"Y:\不同传感器数据\VIIRS\VNP02IMG.A2021182.0418.001.2021182100800.nc" + # + # GF3MDJPath = r'Y:\不同传感器数据\GF-3\GF3_MDJ_SS_024986_E120.8_N35.6_20210509_L1A_VHVV_L10005638033.tar.gz' + # GF4PMIPath = r'Y:\不同传感器数据\GF-4\GF4_PMI_E119.8_N35.3_20210908_L1A0000417337.tar.gz' + # S3OLFilePath = r'Y:\不同传感器数据\SENTINEL-3' \ + # r'\S3B_OL_1_EFR____20210910T022645_20210910T022945_20210911T064342_0179_056_374_2340_LN1_O_NT_002.zip' + # S3SLFilePath = r'Y:\不同传感器数据\SENTINEL-3' \ + # r'\S3A_SL_1_RBT____20210916T020956_20210916T021256_20210917T120953_0179_076_217_2340_LN2_O_NT_004.zip' # 读取 HJ-1 元数据 - hj1_dic = GetHJ1Data(HJ1FilePath) - print(hj1_dic) + # hj1_dic = GetHJ1Data(HJ1FilePath) + # print(hj1_dic) # 读取 JPSS 元数据 - jpss_dic = GetJPSSData(JPSSFilePath) - print(jpss_dic) + # jpss_dic = GetJPSSData(JPSSFilePath) + # print(jpss_dic) # 读取 ZY2 元数据 - zy2_mux_dic, zy2_pan_dic = GetZY02CData(ZY2FilePath) - print(zy2_mux_dic) - print(zy2_pan_dic) + # zy2_mux_dic, zy2_pan_dic = GetZY02CData(ZY2FilePath) + # print(zy2_mux_dic) + # print(zy2_pan_dic) # 读取 ZY3 元数据 - zy3_dic = GetZY3Data(ZY3FilePath) - print(zy3_dic) + # zy3_dic = GetZY3Data(ZY3FilePath) + # print(zy3_dic) # 读取GF-PMS元数据 - pms_mss_dic, pms_pan_dic = GetGFPMSData(GF1PMSPath) - print(pms_mss_dic) - print(pms_pan_dic) + # pms_mss_dic, pms_pan_dic = GetGFPMSData(GF1PMSPath) + # print(pms_mss_dic) + # print(pms_pan_dic) # 读取葵花8元数据 - h8_dic = GetH08Data(H08FilePath) - print(h8_dic) + # h8_dic = GetH08Data(H08FilePath) + # print(h8_dic) # 读取 S2 元数据 - s2_dic = GetSentinel2Data(S2FilePath) - print(s2_dic) + # xml_path = r"D:\1Company\Python\RS_data_Dowload\Google_Download_New\data" + # thumbnail_path = r"D:\1Company\Python\RS_data_Dowload\Google_Download_New\data" + # s2_dic = GetSentinel2Data(S2FilePath, xml_path, thumbnail_path) + # print(s2_dic) # 读取 S1 元数据 - s1_dic = GetSentinel1Data(S1FilePath) - print(s1_dic) + # s1_dic = GetSentinel1Data(S1FilePath) + # print(s1_dic) # 读取 SNPP 元数据 - snpp_dic = GetSNPPData(SNPPFilePath) - print(snpp_dic) + # snpp_dic = GetSNPPData(SNPPFilePath) + # print(snpp_dic) # 读取 GF3 元数据 - gf3_dic = GetGF3MDJData(GF3MDJPath) - print(gf3_dic) + # gf3_dic = GetGF3MDJData(GF3MDJPath) + # print(gf3_dic) # 读取 GF4 元数据 - gf4_pms_dic, gf4_irs_dic = GetGF4PMIData(GF4PMIPath) - print(gf4_pms_dic) - print(gf4_irs_dic) + # gf4_pms_dic, gf4_irs_dic = GetGF4PMIData(GF4PMIPath) + # print(gf4_pms_dic) + # print(gf4_irs_dic) # 读取 S3 OL元数据 - s3ol_dic = GetSentinel3OLData(S3OLFilePath) - print(s3ol_dic) + # s3ol_dic = GetSentinel3OLData(S3OLFilePath) + # print(s3ol_dic) # # 读取 S3 SL元数据 # s3sl_dic = GetSentinel3SLData(S3SLFilePath) # print(s3sl_dic) + + # 读取 S2 元数据 + S2FilePath = r"F:\test\Sentinel\S2A_MSIL1C_20220102T031131_N0301_R075_T50SLG_20220102T050158.SAFE.zip" + xml_path = r"F:\test\Sentinel" + thumbnail_path = r"F:\test\Sentinel" + s2_dic = GetSentinel2Data(S2FilePath, xml_path, thumbnail_path) + print(s2_dic) + + # 读取Landsat 8 数据 + LandsatFilePath = r"F:\test\USGS_data\landsat_8_c1\LC81220342021355LGN00.tar.gz" + thumbnail_path1 = r"F:\test\USGS_data\landsat_8_c1" + txt_path = thumbnail_path1 + Landsat_dic = GetLandsatData(LandsatFilePath, thumbnail_path1, txt_path) + print(Landsat_dic) + + # 读取MODIS数据 + + MODIS_path = r"F:\test\MODIS\MOD11A1\MOD11A1.A2022038.h26v05.006.2022039112456.hdf" # MOD11A1数据信息读取 + MODIS = GetModisData(MODIS_path) + print(MODIS) + + # 读取GOCI数据 + GOCI_FilePath = r"F:\test\GOCI_Data\GOCI_L2\2020-01-01\COMS_GOCI_L2A_GA_20200101001642.CDOM.he5" + GOCI = GetGOCIData(GOCI_FilePath) + print(GOCI) + + # 读取GK2B_GOCI数据 + GK2B_FilePath = r"F:\test\GOCI_Data\GK2_GC2_L2\AC\GK2B_GOCI2_L2_20220208_011530_LA_S010_AC.nc" + GK2B = GetGK2BData(GK2B_FilePath) + print(GK2B) \ No newline at end of file diff --git a/scan_data/__init__.py b/data_meta/__init__.py similarity index 100% rename from scan_data/__init__.py rename to data_meta/__init__.py diff --git a/dms_client.db b/dms_client.db index cd1ede4..ac8ad38 100644 Binary files a/dms_client.db and b/dms_client.db differ diff --git a/requirements.txt b/requirements.txt index 32c5b47..49ca4fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,22 @@ affine==2.3.0 attrs==21.4.0 certifi==2021.10.8 -click==8.0.3 +click==8.0.4 click-plugins==1.1.1 cligj==0.7.2 colorama==0.4.4 -GDAL @ file:///F:/Soft/Python%E5%BC%80%E5%8F%91/windows%E9%80%82%E9%85%8D%E5%BA%93/GDAL-3.4.1-cp310-cp310-win_amd64.whl numpy==1.22.2 pyparsing==3.0.7 pyproj==3.3.0 -rasterio @ file:///F:/Soft/Python%E5%BC%80%E5%8F%91/windows%E9%80%82%E9%85%8D%E5%BA%93/rasterio-1.2.10-cp310-cp310-win_amd64.whl six==1.16.0 snuggs==1.4.7 +flask-cors~=3.0.10 + +APScheduler~=3.8.1 +Flask~=2.0.3 +Pillow~=9.0.1 +redis~=4.1.4 +h5py~=3.6.0 +requests~=2.27.1 +dict2xml~=1.7.1 +xmltodict~=0.12.0 \ No newline at end of file diff --git a/run.py b/run.py new file mode 100644 index 0000000..536819a --- /dev/null +++ b/run.py @@ -0,0 +1,8 @@ + +from common.config.factory import create_app + +app = create_app() + +if __name__ == '__main__': + print(app.url_map) + app.run(host="192.168.2.105", port=9550, debug=True, use_reloader=False) diff --git a/scan_data/__pycache__/GetMetaInfo.cpython-39.pyc b/scan_data/__pycache__/GetMetaInfo.cpython-39.pyc deleted file mode 100644 index 5e6cf27..0000000 Binary files a/scan_data/__pycache__/GetMetaInfo.cpython-39.pyc and /dev/null differ diff --git a/scan_data/__pycache__/__init__.cpython-39.pyc b/scan_data/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index 0af127e..0000000 Binary files a/scan_data/__pycache__/__init__.cpython-39.pyc and /dev/null differ diff --git a/scan_data/__pycache__/example.cpython-39.pyc b/scan_data/__pycache__/example.cpython-39.pyc deleted file mode 100644 index d9cf58b..0000000 Binary files a/scan_data/__pycache__/example.cpython-39.pyc and /dev/null differ diff --git a/scan_data/example.py b/scan_data/example.py deleted file mode 100644 index 101bb41..0000000 --- a/scan_data/example.py +++ /dev/null @@ -1,165 +0,0 @@ -from xml.dom import minidom -from osgeo import gdal -from PIL import Image -import numpy as np -import os - - -def uint16to8(bands, lower_percent=0.001, higher_percent=99.999): - """ - 拉伸图像：图片16位转8位 - :param bands: 输入栅格数据 - :param lower_percent: 最低百分比 - :param higher_percent: 最高百分比 - :return: - """ - out = np.zeros_like(bands, dtype=np.uint8) - n = bands.shape[0] - for i in range(n): - a = 0 # np.min(band) - b = 255 # np.max(band) - c = np.percentile(bands[i, :, :], lower_percent) - d = np.percentile(bands[i, :, :], higher_percent) - t = a + (bands[i, :, :] - c) * (b - a) / (d - c) - t[t < a] = a - t[t > b] = b - out[i, :, :] = t - return out - - -def createXML(metadata, xlm_file): - """ - 创建xlm文件并写入字典 - :param metadata: 元数据信息 - :param xlm_file: xlm文件 - :return: - """ - # 创建一个空的文档 - document = minidom.Document() # 创建DOM文档对象 - # 创建一个根节点对象 - root = document.createElement('ProductMetaData') - # 设置根节点的属性 - # root.setAttribute('', '') - # 将根节点添加到文档对象中 - document.appendChild(root) - # 字典转xml - for key in metadata: - # 创建父节点 - node_name = document.createElement(key) - # 给父节点设置文本 - node_name.appendChild(document.createTextNode(str(metadata[key]))) - # 将各父节点添加到根节点 - root.appendChild(node_name) - # 写入xlm文档 - with open(xlm_file, 'w', encoding='utf-8') as f: - document.writexml(f, indent='\t', newl='\n', addindent='\t', encoding='utf-8') - f.close() - - -def GetJPSSData(in_file, xml_path, thumbnail_path): - """ - 获取联合极轨卫星系统（JPSS-1）元数据：NOAA-20(Joint Polar Satellite System spacecraft) - :param xml_path: - :param thumbnail_path: - :param in_file: - :return: 元数据字典 - """ - try: - # 生成缩略图 - in_path, basename = os.path.split(in_file) - ThumbnailName = os.path.splitext(basename)[0] + "_thumb.jpg" - ThumbnailPath = os.path.join(thumbnail_path, ThumbnailName) - - gdal.SetConfigOption("GDAL_FILENAME_IS_UTF8", "YES") - in_datasets = gdal.Open(in_file) - meta_data = in_datasets.GetMetadata() - # 取出子数据集 - datasets = in_datasets.GetSubDatasets() - red_data = gdal.Open(datasets[0][0]).ReadAsArray() - nir_data = gdal.Open(datasets[3][0]).ReadAsArray() - swir_data = gdal.Open(datasets[9][0]).ReadAsArray() - img_data = np.array([red_data, nir_data, swir_data]) - img_data = uint16to8(img_data) - # Array转Image - img_data2 = np.transpose(img_data, (1, 2, 0)) - img_data2 = img_data2[:, :, ::-1] - img = Image.fromarray(img_data2) - # 压缩图片大小 - if img_data.shape[1] > img_data.shape[2]: - width = 512 - height = int(width / img_data.shape[1] * img_data.shape[2]) - else: - height = 512 - width = int(height / img_data.shape[1] * img_data.shape[2]) - img.thumbnail((width, height)) - img.save(ThumbnailPath, "PNG") - - # 释放内存 - del in_datasets - del img_data - del img_data2 - del img - - # 生成XML文件 - xmlFileName = os.path.splitext(basename)[0] + ".xml" - xmlPath = os.path.join(xml_path, xmlFileName) - createXML(meta_data, xmlPath) - - # 产品日期 - ProductionTime = meta_data['ProductionTime'] - StartTime = meta_data['StartTime'] - EndTime = meta_data['EndTime'] - - # 其他信息 - ImageGSD = str(meta_data['LongName']).split(" ")[-1] - Bands = str(meta_data['title']).split(" ")[1] - - # 中心经纬度 - productUpperLeftLat = meta_data['NorthBoundingCoordinate'] # 左上纬度 - productUpperLeftLong = meta_data['WestBoundingCoordinate'] # 左上经度 - productUpperRightLat = meta_data['NorthBoundingCoordinate'] # 右上纬度 - productUpperRightLong = meta_data['EastBoundingCoordinate'] # 右上经度 - productLowerLeftLat = meta_data['SouthBoundingCoordinate'] # 左下纬度 - productLowerLeftLong = meta_data['WestBoundingCoordinate'] # 左下经度 - productLowerRightLat = meta_data['SouthBoundingCoordinate'] # 右下纬度 - productLowerRightLong = meta_data['EastBoundingCoordinate'] # 右下纬度 - - # 边界几何 - boundaryGeomStr = f'POLYGON(({productUpperLeftLong} {productUpperLeftLat},' \ - f'{productUpperRightLong} {productUpperRightLat},' \ - f'{productLowerRightLong} {productLowerRightLat},' \ - f'{productLowerLeftLong} {productLowerLeftLat},' \ - f'{productUpperLeftLong} {productUpperLeftLat}))' - - # 构建字典 - jpss_dict = {"ProduceTime": ProductionTime, - "StartTime": StartTime, - "EndTime": EndTime, - "CloudPercent": "", - # "TopLeftLatitude": productUpperLeftLat, - # "TopLeftLongitude": productUpperLeftLong, - # "TopRightLatitude": productUpperRightLat, - # "TopRightLongitude": productUpperRightLong, - # "BottomLeftLatitude": productLowerLeftLat, - # "BottomLeftLongitude": productLowerLeftLong, - # "BottomRightLatitude": productLowerRightLat, - # "BottomRightLongitude": productLowerRightLong, - "boundaryGeomStr": boundaryGeomStr, - "bands": Bands, - "ImageGSD": ImageGSD, - "ProjectedCoordinates": "", - "CollectionCode": "", - "ThumbnailPath": ThumbnailPath, - "ThumbnailName": ThumbnailName, - "xmlPath": xmlPath, - "xmlFileName": xmlFileName, - "DirectoryDepth": "day"} - - # 判断字典是否为空 - if not jpss_dict: - return {"code": -1, "msg": "没有满足条件的数据字典..."} - print(jpss_dict) - return jpss_dict - except Exception as e: - print(str(e)) - return {"code": -1, "msg": str(e)} \ No newline at end of file diff --git a/scan_data/receive_queue.py b/scan_data/receive_queue.py deleted file mode 100644 index 77cbc2a..0000000 --- a/scan_data/receive_queue.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/python3 -# coding= utf-8 - -import sqlite3 - - -# @param text 文本 -# @return True：是，False：不是 -def __string(text): - return True if isinstance(text, str) else False - - -# 检查文本类型是否为浮点型 -# @param text 文本 -# @return True：是，False：不是 -def __float(text): - if __string(text): - try: - return True if float("{0}".format(text)) else False - except Exception: - return False - else: - return True if isinstance(text, float) else False - - -# 检查文本类型是否为浮点型 -# @param text 文本 -# @return True：是，False：不是 -def __int(text): - if __string(text): - try: - return True if float("{0}".format(text)) else False - except Exception: - return False - else: - return True if isinstance(text, float) else False - - -# 检查文本类型是否为数字型 -# @param text 文本 -# @return True：是，False：不是 -def __number(text): - return True if re.search("[^0-9]", text) == None else False - - -def func_year(s): - print('func_year:', s) - - -def func_month(s): - print('func_month:', s) - - -def create_data_collection_info_table(): - con = sqlite3.connect("../dms_client.db") - cur = con.cursor() - sql = "CREATE TABLE IF NOT EXISTS data_collection_info(id INTEGER PRIMARY KEY,collection_code TEXT,function_name TEXT,describe TEXT)" - cur.execute(sql) - # ①：添加单条数据 - data = "1,'Desire',5,'test'" - cur.execute('INSERT INTO data_collection_info VALUES (%s)' % data) - # ②：添加单条数据 - cur.execute("INSERT INTO data_collection_info values(?,?,?,?)", (6, "zgq", 20, 'test')) - # ③：添加多条数据 - cur.executemany('INSERT INTO data_collection_info VALUES (?,?,?,?)', - [(3, 'name3', 19, 'test'), (4, 'name4', 26, 'test')]) - cur.execute("UPDATE data_collection_info SET collection_code=? WHERE id=?", ('test1', 19)) - con.commit() - # 关闭游标 - cur.close() - # 断开数据库连接 - con.close() - - -# 动态调用函数 -# @param tag 标签名 -# @param text 文本 -# @return True:OK, False:NG -def item_check(func_name, text): - if type and text: - try: - # 调用导入模块中的函数，并传参 - return eval("__{0}".format(func_name))(text) - except Exception: - return False - else: - return False - - -if __name__ == '__main__': - result = item_check("num", 123.23) - print(result) - strs = ['year', 'month'] - for s in strs: - globals().get('func_%s' % s)(s) diff --git a/scheduled_task/queue_storage.py b/scheduled_task/queue_storage.py new file mode 100644 index 0000000..dee5e45 --- /dev/null +++ b/scheduled_task/queue_storage.py @@ -0,0 +1,119 @@ +# coding: utf-8 +# Author：tajochen + +import time + +import requests + +from application.settings import Config +from common.tools.dms import dms_login, dms_task_record +from util.upload_client import upload_file_client + +from util.http_util import httpUtil + + +# 动态调用函数 +# @param tag 标签名 +# @param text 文本 +# @return True:OK, False:NG +def item_check(func_name, text): + if type and text: + try: + # 调用导入模块中的函数，并传参 + return eval("__{0}".format(func_name))(text) + except Exception: + return False + else: + return False + + +# 动态调用遥感函数 +# @param tag 标签名 +# @param text 文本 +# @return True:OK, False:NG +def analysis_remote_sensing_data(func_name, collection_code, path): + """ + 解析JPSS-VJ102元数据 + :return: + """ + print('开始扫描VJ102IMG数据集') + #collectionCode = 'VJ102IMG' + # 用户登录 + token_s = dms_login() + # 判断定时任务是否在进行 + task = dms_task_record(token_s, collection_code) + # 如果不是空说明正在进行 + if task is not None and len(task) > 0: + return 1 + + file_total_size = "" + file_total_name = "" + xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 + ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 + + # 解析遥感数据文件（demo） + remote_sensing_data_dic = globals().get('func_%s' % func_name)(path, xmlPath, ThumbnailPath) + + # 配置文件服务器参数 + url = Config.DFS_UPLOAD_URL + files = {'file': open(remote_sensing_data_dic['xmlPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collection_code} + # 上传生成的xml文件到文件服务器 + xml = requests.post(url, data=options, files=files) + + url = Config.DFS_UPLOAD_URL + files = {'file': open(remote_sensing_data_dic['ThumbnailPath'], 'rb')} + options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collection_code} + # 上传生成的xml文件到文件服务器 + ThumbnailName = requests.post(url, data=options, files=files) + + CollectionCode = remote_sensing_data_dic['CollectionCode'] + DirectoryDepth = remote_sensing_data_dic['DirectoryDepth'] + StartTime = remote_sensing_data_dic['StartTime'] + # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) + uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collection_code) + + StartTime = time.mktime(time.strptime(remote_sensing_data_dic['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) + EndTime = time.mktime(time.strptime(remote_sensing_data_dic['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) + + # 入库遥感数据 + res_data = httpUtil(url=Config.RESING_DATA_URL, + data={"collectionCode": collection_code, "shootingTimeStartTs": StartTime, + "shootingTimeEndTs": EndTime, + "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], + "fileSize": uc['file_size'], "cloudCoverage": remote_sensing_data_dic['CloudPercent'], + "metaInformationFile": xml.json()['path'], + "thumbnailFile": ThumbnailName.json()['path'], + "remarks": "", "boundaryGeomStr": remote_sensing_data_dic['boundaryGeomStr']}, + token=token_s).post_no_patam_herder() + print(res_data.json()['data']) + + file_total_size = file_total_size + str(uc['file_size']) + file_total_size = file_total_size + "," + + file_total_name = file_total_name + uc['fileName'] + file_total_name = file_total_name + "," + + # 添加遥感数据归档任务 + res = httpUtil(url=Config.DATA_TASK_URL, + data={"clientCode": "client1", "collectionCode": collection_code, "storageFileList": file_total_name, + "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() + task_code = res.json()['data'] + + # 结束遥感数据归档任务 + header = {"Authorization": token_s} + res = requests.post(url=Config.DATA_END_TASK_URL, + + params={"taskCode": task_code}, headers=header).json() + if res_data.json()['status'] == 0: + return 0 + else: + return 2 + + +if __name__ == "__main__": + result = item_check("num", 123.23) + print(result) + strs = ['year', 'month'] + for s in strs: + globals().get('func_%s' % s)(s) diff --git a/upload/__init__.py b/upload/__init__.py deleted file mode 100644 index 00d56c8..0000000 --- a/upload/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -Author : XinYi Song -Time : 2021/12/3 11:17 -Desc: -""" diff --git a/upload/__pycache__/__init__.cpython-39.pyc b/upload/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index 4831f2e..0000000 Binary files a/upload/__pycache__/__init__.cpython-39.pyc and /dev/null differ diff --git a/upload/__pycache__/upload_client.cpython-39.pyc b/upload/__pycache__/upload_client.cpython-39.pyc deleted file mode 100644 index 43cb636..0000000 Binary files a/upload/__pycache__/upload_client.cpython-39.pyc and /dev/null differ diff --git a/util/UDP_Receive.py b/util/UDP_Receive.py index 5fa56f2..46e8dfc 100644 --- a/util/UDP_Receive.py +++ b/util/UDP_Receive.py @@ -4,12 +4,17 @@ import socket import sys import struct +import json # 本机信息 +import time + +from util.simple_sqlite3_tool import SimpleSQLite3Tool + host_ip = socket.gethostbyname(socket.gethostname()) # 组播组IP和端口 -mcast_group_ip = '239.255.255.252' -mcast_group_port = 5678 +mcast_group_ip = '224.1.1.1' +mcast_group_port = 2234 def receiver(): @@ -34,6 +39,23 @@ def receiver(): while True: try: data, address = sock.recvfrom(4096) + sql_tool = SimpleSQLite3Tool("../dms_client.db") + data2 = json.loads(data) + # 若组播数据不正确 + if len(data2) < 3: + return + # 若数据已存在 + res1 = sql_tool.query("select * from data_list where collection_code=? and file_name=?;", (data2.collectionCode,data2.filename)) + if len(res1) != 0: + return + print(res1) + # 存入归档数据 + current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + res2 = sql_tool.execute("insert into data_list " + "(file_path,file_name,collection_code,archive_status,created_time) values (?,?,?,?,?);", + (data2.file_path, data2.file_name, data2.collection_code, 0, current_time)) + print(res2) + sql_tool.close() except socket.error as e: print(f"while receive message error occur:{e}") else: diff --git a/util/__pycache__/file_store_path.cpython-39.pyc b/util/__pycache__/file_store_path.cpython-39.pyc deleted file mode 100644 index a07dca2..0000000 Binary files a/util/__pycache__/file_store_path.cpython-39.pyc and /dev/null differ diff --git a/util/__pycache__/http_file_upload.cpython-39.pyc b/util/__pycache__/http_file_upload.cpython-39.pyc deleted file mode 100644 index 5e91e86..0000000 Binary files a/util/__pycache__/http_file_upload.cpython-39.pyc and /dev/null differ diff --git a/util/__pycache__/remote_sensing_util.cpython-39.pyc b/util/__pycache__/remote_sensing_util.cpython-39.pyc deleted file mode 100644 index 9adc1e8..0000000 Binary files a/util/__pycache__/remote_sensing_util.cpython-39.pyc and /dev/null differ diff --git a/util/__pycache__/scan_file_util.cpython-39.pyc b/util/__pycache__/scan_file_util.cpython-39.pyc deleted file mode 100644 index 0e3ac93..0000000 Binary files a/util/__pycache__/scan_file_util.cpython-39.pyc and /dev/null differ diff --git a/util/file_store_path.py b/util/file_store_path.py deleted file mode 100644 index 8afd439..0000000 --- a/util/file_store_path.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Author : XinYi Song -Time : 2021/11/4 9:27 -Desc: -""" -import datetime -import os -import time - - -def file_store_path(time_stamp): - """ - :param time_stamp: 时间戳类型时间 - :return: - """ - now = int(round(time_stamp * 1000)) - t = time.localtime(now / 1000) - return os.path.join('E:/data/upload', str(t[0]), str(t[1]), str(t[2])) - - -def file_store_path_year(data_str_time, upload_path, conllection_code): - """ - 目录到年 - :param upload_path: - :param data_str_time: 字符串类型 - :return: - """ - t = time.strptime(data_str_time, '%Y-%m-%d %H:%M:%S') - return os.path.join(upload_path, conllection_code, str(t[0])) - - -def file_store_path_month(data_str_time, upload_path, conllection_code): - """ - 目录到月 - :param upload_path: - :param data_str_time: - :return: - """ - t = time.strptime(data_str_time, '%Y-%m-%d %H:%M:%S') - return os.path.join(upload_path, conllection_code, str(t[0]), str(t[1])) - - -def file_store_path_day(data_str_time, upload_path, conllection_code): - """ - 目录到日 - :param upload_path: - :param data_str_time: 字符串类型的时间 - :return: - """ - t = time.strptime(data_str_time, '%Y-%m-%d %H:%M:%S') - return os.path.join(upload_path, conllection_code, str(t[0]), str(t[1]), str(t[2])) - - -if __name__ == '__main__': - # time_stamp1 = time.time() - # print(time_stamp1) - str_time = '2020-06-08 09:33:07' - t = time.strptime(str_time, '%Y-%m-%d %H:%M:%S') - # path = os.path.join('../upload', str(t[0]), str(t[1]), str(t[2])) - # if not os.path.exists(path): - # os.makedirs(path) - # print(path) - # time_stamp = float(time_stamp1) - # now = int(round(time_stamp * 1000)) - # t = time.localtime(now / 1000) - # print(t) - - # list1 = ['张三', '李四'] - # token_s = dms_login() - # dms_list = dms_sensing_data(token_s) - # 数据库返回值 - # list2 = ['张三', '李四'] - - # d = [y for y in list2 if y not in list1] - # if d is None or len(d) == 0: - # print("d为空") - # else: - # print(d) - # file_dir = 'C:/Users/HP/Desktop/数管/' - # dir_list = os.listdir(file_dir) - # print(dir_list) - # timestring = '2016-12-21 10:22:56' - # print(time.mktime(time.strptime(timestring, '%Y-%m-%d %H:%M:%S'))) # 1482286976.0 - print(t) diff --git a/util/file_util.py b/util/file_util.py deleted file mode 100644 index 4559a7a..0000000 --- a/util/file_util.py +++ /dev/null @@ -1,1050 +0,0 @@ -""" -Author : XinYi Song -Time : 2022/2/12 17:13 -Desc: -""" -import os -import time - -import requests - -from application.settings import Config -from common.tools.dms import dms_login, dms_task_record, dms_sensing_data -from scan_data.GetMetaInfo import GetGFPMSData, GetGF3MDJData, GetH08Data, GetSentinel1Data, GetSentinel2Data, \ - GetSentinel3OLData, GetHJ1Data, GetZY3Data, GetSNPPData -from scan_data.example import GetJPSSData -from upload.upload_client import upload_file_client -from util.http_file_upload import upload_client -from util.http_util import httpUtil -from util.remote_sensing_util import gf4_pmi_001 -from util.xml_util import dict_to_xml - - -def list_dir(file_dir): - """ - 通过 listdir 得到的是仅当前路径下的文件名，不包括子目录中的文件，如果需要得到所有文件需要递归 - """ - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = '' # 解析出的xml文件保存的路径 - ThumbnailPath = '' # 解析出的缩略图保存的路径 - for cur_file in d: - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - file_name = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.tiff' - xml_path = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml' - # 解析遥感数据文件（demo） - gf4_sensing = gf4_pmi_001(file_name, xml_path) - file_name = os.path.basename(path).split('.')[0] - xml_path = 'D:/file/work/pythonyuanma/dms_management/xml/' + file_name + '.xml' - # 将遥感数据写入xml中，并保存本地文件夹中 - dict_to_xml(gf4_sensing, xml_path) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(xml_path, 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项 - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(gf4_sensing['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项 - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = gf4_sensing['CollectionCode'] - DirectoryDepth = gf4_sensing['DirectoryDepth'] - StartTime = gf4_sensing['StartTime'] - uc = upload_client(path, DirectoryDepth, StartTime) - - StartTime = time.mktime(time.strptime(gf4_sensing['StartTime'], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(gf4_sensing['EndTime'], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": "GF4_PMI_001", "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": gf4_sensing['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": gf4_sensing['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": "GF4_PMI_001", "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_VJ102_dir(collectionCode, path): - """ - 解析JPSS-VJ102元数据 - :return: - """ - print('开始扫描VJ102IMG数据集') - #collectionCode = 'VJ102IMG' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = JPSSData_dict['CollectionCode'] - DirectoryDepth = JPSSData_dict['DirectoryDepth'] - StartTime = JPSSData_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_VJ103_dir(collectionCode, path): - """ - 解析JPSS-VJ103元数据 - :param file_dir: - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\JPSS' - print('开始扫描VJ103IMG数据集') - #collectionCode = 'VJ103IMG' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = JPSSData_dict['CollectionCode'] - DirectoryDepth = JPSSData_dict['DirectoryDepth'] - StartTime = JPSSData_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_GF1_PMS2_dir(collectionCode, path): - """ - 获取高分 PMS卫星元数据 - :return: - """ - - print('开始扫描GF1_PMS2_001数据集') - #collectionCode = 'GF1_PMS2_001' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - GFPMS_dict = GetGFPMSData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - print(GFPMS_dict['xmlPath']) - files = {'file': open(GFPMS_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GFPMS_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GFPMS_dict['CollectionCode'] - DirectoryDepth = GFPMS_dict['DirectoryDepth'] - StartTime = GFPMS_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(GFPMS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GFPMS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GFPMS_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GFPMS_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_GF3MDJ_dir(collectionCode, path): - """ - 获取高分3号MDJ（GF-3 MDJ）卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - print('开始扫描GF3_MDJ_SS数据集') - #collectionCode = 'GF3_MDJ_SS' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - GF3_MDJ_SS_dict = GetGF3MDJData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GF3_MDJ_SS_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GF3_MDJ_SS_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GF3_MDJ_SS_dict['CollectionCode'] - DirectoryDepth = GF3_MDJ_SS_dict['DirectoryDepth'] - StartTime = GF3_MDJ_SS_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GF3_MDJ_SS_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GF3_MDJ_SS_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_H08_dir(collectionCode, path): - """ - 获取葵花8卫星元数据 - :return: - """ - print('开始扫描H08数据集') - #collectionCode = 'NC_H08' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - GetH08_dict = GetH08Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetH08_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetH08_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetH08_dict['CollectionCode'] - DirectoryDepth = GetH08_dict['DirectoryDepth'] - StartTime = GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime( - time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime( - time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetH08_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetH08_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_Sentinel1_dir(collectionCode, path): - """ - 获取哨兵1号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - print('开始扫描Sentinel1数据集') - #collectionCode = 'S1A_IW_GRDH' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - Sentinel1_dict = GetSentinel1Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel1_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel1_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = Sentinel1_dict['CollectionCode'] - DirectoryDepth = Sentinel1_dict['DirectoryDepth'] - StartTime = Sentinel1_dict['ProduceTime'][0:10] + ' ' + Sentinel1_dict['ProduceTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(Sentinel1_dict['StartTime'][0:10] + ' ' + Sentinel1_dict['StartTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime( - time.strptime(Sentinel1_dict['StopTime'][0:10] + ' ' + Sentinel1_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": Sentinel1_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": Sentinel1_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_Sentinel2_dir(collectionCode, path): - """ - 获取哨兵2号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - print('开始扫描Sentinel2数据集') - #collectionCode = 'S2B' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - Sentinel2_dict = GetSentinel2Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel2_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel2_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = Sentinel2_dict['CollectionCode'] - DirectoryDepth = Sentinel2_dict['DirectoryDepth'] - StartTime = Sentinel2_dict['ProduceTime'][0:10] + ' ' + Sentinel2_dict['ProduceTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(Sentinel2_dict['StartTime'][0:10] + ' ' + Sentinel2_dict['StartTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime( - time.strptime(Sentinel2_dict['StopTime'][0:10] + ' ' + Sentinel2_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": Sentinel2_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": Sentinel2_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_Sentinel3OL_dir(collectionCode, path): - """ - 获取哨兵3号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - print('开始扫描Sentinel3数据集') - #collectionCode = 'Sentinel3_OLCI' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - Sentinel3OL_dict = GetSentinel3OLData(path, xmlPath, ThumbnailPath) - print(Sentinel3OL_dict) - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel3OL_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel3OL_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = Sentinel3OL_dict['CollectionCode'] - DirectoryDepth = Sentinel3OL_dict['DirectoryDepth'] - StartTime = Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime( - time.strptime(Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(Sentinel3OL_dict['StopTime'][0:10] + ' ' + Sentinel3OL_dict['StopTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": Sentinel3OL_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": Sentinel3OL_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_HJ1_dir(collectionCode, path): - """ - 获取环境1号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - #file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\HJ-1' - print('开始扫描环境1号数据集') - #collectionCode = 'HJ-1' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - GetHJ1Data_dict = GetHJ1Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetHJ1Data_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetHJ1Data_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetHJ1Data_dict['CollectionCode'] - DirectoryDepth = GetHJ1Data_dict['DirectoryDepth'] - StartTime = GetHJ1Data_dict['ProductTime'][0:10] + ' ' + GetHJ1Data_dict['ProductTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime( - time.strptime(GetHJ1Data_dict['StartTime'][0:10] + ' ' + GetHJ1Data_dict['StartTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime( - time.strptime(GetHJ1Data_dict['EndTime'][0:10] + ' ' + GetHJ1Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetHJ1Data_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetHJ1Data_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_ZY3_dir(collectionCode, path): - """ - 获取资源3号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - #file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\ZY-3' - print('开始扫描资源3号数据集') - #collectionCode = 'ZY-3' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - GetZY3Data_dict = GetZY3Data(path, xmlPath, ThumbnailPath) - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetZY3Data_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetZY3Data_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetZY3Data_dict['CollectionCode'] - DirectoryDepth = GetZY3Data_dict['DirectoryDepth'] - StartTime = GetZY3Data_dict['ProduceTime'][0:10] + ' ' + GetZY3Data_dict['ProduceTime'][11:19] - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime( - time.strptime(GetZY3Data_dict['StartTime'][0:10] + ' ' + GetZY3Data_dict['StartTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime( - time.strptime(GetZY3Data_dict['EndTime'][0:10] + ' ' + GetZY3Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetZY3Data_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetZY3Data_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -def scan_SNPP_dir(collectionCode, path): - """ - 获取资源3号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - #file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\VIIRS' - print('开始扫描VNP02IMG数据集') - #collectionCode = 'VNP02IMG' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return 1 - - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - - # 解析遥感数据文件（demo） - GetSNPPData_dict = GetSNPPData(path, xmlPath, ThumbnailPath) - print(GetSNPPData_dict) - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetSNPPData_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetSNPPData_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetSNPPData_dict['CollectionCode'] - DirectoryDepth = GetSNPPData_dict['DirectoryDepth'] - StartTime = GetSNPPData_dict['ProductionTime'][0:10] + ' ' + GetSNPPData_dict['ProductionTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime, collectionCode) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime( - time.strptime(GetSNPPData_dict['StartTime'][0:10] + ' ' + GetSNPPData_dict['StartTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GetSNPPData_dict['EndTime'][0:10] + ' ' + GetSNPPData_dict['EndTime'][11:19], - '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetSNPPData_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetSNPPData_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - - - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - if res_data.json()['status'] == 0: - return 0 - else: - return 2 - - -if __name__ == '__main__': - # file_dir = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_001/sensingdata' - # list_dir(file_dir) - # file_dir = 'E:/数管' - # - scan_VJ102_dir() - # - # scan_VJ103_dir() - - # scan_GF1_PMS2_dir() - # scan_GF3MDJ_dir() - # scan_H08_dir() - # scan_Sentinel1_dir() - # scan_Sentinel2_dir() - # scan_Sentinel3OL_dir() - # scan_HJ1_dir() - # scan_ZY3_dir() - # scan_SNPP_dir() diff --git a/util/find_GPS_image.py b/util/find_GPS_image.py deleted file mode 100644 index 7b144e6..0000000 --- a/util/find_GPS_image.py +++ /dev/null @@ -1,81 +0,0 @@ -import exifread -import re -import json -import requests - -def latitude_and_longitude_convert_to_decimal_system(*arg): - """ - 经纬度转为小数, param arg: - :return: 十进制小数 - """ - return float(arg[0]) + ((float(arg[1]) + (float(arg[2].split('/')[0]) / float(arg[2].split('/')[-1]) / 60)) / 60) - -def find_GPS_image(pic_path): - GPS = {} - date = '' - with open(pic_path, 'rb') as f: - tags = exifread.process_file(f) - for tag, value in tags.items(): - if re.match('GPS GPSLatitudeRef', tag): - GPS['GPSLatitudeRef'] = str(value) - elif re.match('GPS GPSLongitudeRef', tag): - GPS['GPSLongitudeRef'] = str(value) - elif re.match('GPS GPSAltitudeRef', tag): - GPS['GPSAltitudeRef'] = str(value) - elif re.match('GPS GPSLatitude', tag): - try: - match_result = re.match('\[(\w*),(\w*),(\w.*)/(\w.*)\]', str(value)).groups() - GPS['GPSLatitude'] = int(match_result[0]), int(match_result[1]), int(match_result[2]) - except: - deg, min, sec = [x.replace(' ', '') for x in str(value)[1:-1].split(',')] - GPS['GPSLatitude'] = latitude_and_longitude_convert_to_decimal_system(deg, min, sec) - elif re.match('GPS GPSLongitude', tag): - try: - match_result = re.match('\[(\w*),(\w*),(\w.*)/(\w.*)\]', str(value)).groups() - GPS['GPSLongitude'] = int(match_result[0]), int(match_result[1]), int(match_result[2]) - except: - deg, min, sec = [x.replace(' ', '') for x in str(value)[1:-1].split(',')] - GPS['GPSLongitude'] = latitude_and_longitude_convert_to_decimal_system(deg, min, sec) - elif re.match('GPS GPSAltitude', tag): - GPS['GPSAltitude'] = str(value) - elif re.match('.*Date.*', tag): - date = str(value) - return {'GPS_information': GPS, 'date_information': date} - -def find_address_from_GPS(GPS): - """ - 使用Geocoding API把经纬度坐标转换为结构化地址。 - :param GPS: - :return: - """ - secret_key = 'zbLsuDDL4CS2U0M4KezOZZbGUY9iWtVf' - if not GPS['GPS_information']: - return '该照片无GPS信息' - lat, lng = GPS['GPS_information']['GPSLatitude'], GPS['GPS_information']['GPSLongitude'] - baidu_map_api = "http://api.map.baidu.com/geocoder/v2/?ak={0}&callback=renderReverse&location={1},{2}s&output=json&pois=0".format( - secret_key, lat, lng) - response = requests.get(baidu_map_api) - content = response.text.replace("renderReverse&&renderReverse(", "")[:-1] - baidu_map_address = json.loads(content) - formatted_address = baidu_map_address["result"]["formatted_address"] - province = baidu_map_address["result"]["addressComponent"]["province"] - city = baidu_map_address["result"]["addressComponent"]["city"] - district = baidu_map_address["result"]["addressComponent"]["district"] - return formatted_address,province,city,district - -pic_path = 'D:\pythonjob\pic-time&location\DJI_0001.jpg' - -GPS_info = find_GPS_image(pic_path) -address = find_address_from_GPS(GPS=GPS_info) -#print(GPS_info) -#print(address) - -x = list(GPS_info.values()) -#print(x) -time = x[1] -gps_dict_formate = x[0] -y = list(gps_dict_formate.values()) -information = '拍照时间：'+time+'，拍照地点：'+str(address[0])+'（经度:'+str(y[2])+' '+str(y[3])+'，纬度：'+str(y[0])+' '+str(y[1])+'，高度：'+str(y[5])+'米）' - -print(pic_path) -print(information) diff --git a/util/http_file_upload.py b/util/http_file_upload.py deleted file mode 100644 index d2f1a7e..0000000 --- a/util/http_file_upload.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -Author : XinYi Song -Time : 2021/11/3 14:29 -Desc: -""" -from util.file_store_path import file_store_path_day, file_store_path_year, file_store_path_month - -""" -实现文件断点续传 -""" -import sys -import os -from hashlib import md5 - -FILE_DIR = os.path.dirname(__file__) - -BASE_DIR = os.path.dirname(os.path.dirname(__file__)) -home = os.path.join(BASE_DIR, "E:/data/upload") - - -# 定义一个函数，计算进度条 -def bar(num=1, sum=100): - rate = float(num) / float(sum) - rate_num = int(rate * 100) - temp = '\r%d %%' % rate_num - sys.stdout.write(temp) - - -def md5_file(name): - m = md5() - a_file = open(name, 'rb') #需要使用二进制格式读取文件内容 - m.update(a_file.read()) - a_file.close() - return m.hexdigest() - - -def upload_client(local_path, depth, dateTime, conllection_code): - global file_path - while True: - file_byte_size = os.stat(local_path).st_size # 获取文件的大小 - file_name = os.path.basename(local_path) # 设置文件名 - md5 = md5_file(local_path) - - has_sent = 0 - file_obj = open(local_path, 'rb') # 对文件进行读操作 - file_obj.seek(has_sent) # 调整指针 - if depth == 'year': - file_path = file_store_path_year(dateTime, home, conllection_code) - if not os.path.exists(file_path): - os.makedirs(file_path) - if depth == 'month': - file_path = file_store_path_month(dateTime, home, conllection_code) - if not os.path.exists(file_path): - os.makedirs(file_path) - if depth == 'day': - file_path = file_store_path_day(dateTime, home, conllection_code) - if not os.path.exists(file_path): - os.makedirs(file_path) - path = os.path.join(file_path, file_name) - has_received = 0 - - # 首先判断该路径下是否已存在文件 - if os.path.exists(path): - f = open(path, 'wb') - else: - f = open(path, 'wb') - - while has_sent < file_byte_size: - # 读出数据 - data = file_obj.read(1024) - try: - # 写入数据 - f.write(data) - has_received += len(data) - if not data: - raise Exception - except Exception: - flag = False - break - has_sent += len(data) - bar(has_sent, file_byte_size) # 进度条 - print("文件上传成功！") - file_obj.close() - f.close() - file_dict = {'fileName': file_name, 'md5': md5, 'file_size': file_byte_size, 'file_path': file_path, 'type': 'ok'} - return file_dict diff --git a/util/remote_sensing_util.py b/util/remote_sensing_util.py deleted file mode 100644 index 18440ff..0000000 --- a/util/remote_sensing_util.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Author : XinYi Song -Time : 2021/11/4 16:59 -Desc: -""" -import rasterio -import requests - -from util.xml_util import xml_to_dict, dict_to_xml - - -def gf4_pmi_001(file_name, xml_name): - """ - - :param file_name: 扫描文件传过来的遥感数据源文件的路径 - :param xmlPath: 解析出的xml文件存储的路径 - :param ThumbnailPath: 解析出的缩略图文件存储的路径 - :return: - """ - file_path = 'E:/sensing/GF4_PMI_001/' - with rasterio.open(file_path+file_name, 'r') as ds: - # 存放xml，缩略图的文件夹，由根文件夹+数据集代码命名的文件夹组成 - print('该栅格数据的基本数据集信息:') - CollectionCode = 'GF4_PMI_001' # 数据集代码 - DataFormat = ds.driver # DataFormat 数据格式 - NumberBands = ds.count # NumberBands 波段数目 - ImageWidth = ds.width # ImageWidth 影像宽度 - ImageHeight = ds.height # ImageHeight 影像高度 - GeographicScope = ds.bounds # GeographicScope 地理范围 - ReflectionParameter = ds.transform # ReflectionParameter 反射变换参数（六参数模型） - ProjectionDefinition = ds.crs # ProjectionDefinition 投影定义 - # print(CRS.from_epsg(4326)) - # 获取第一个波段数据，跟GDAL一样索引从1开始 - # 直接获得numpy.ndarray类型的二维数组表示，如果read()函数不加参数，则得到所有波段（第一个维度是波段） - band1 = ds.read(1) - FirstBindMax = band1.max() # FirstBindMax 第一波段的最大值 - - FirstBindMin = band1.min() # FirstBindMin 第一波段的最小值 - FirstBindAverage = band1.mean() # FirstBindAverage 第一波段的平均值 - # 根据地理坐标得到行列号 - x, y = (ds.bounds.left + 300, ds.bounds.top - 300) # 距离左上角东300米，南300米的投影坐标 - row, col = ds.index(x, y) # 对应的行列号 - print(f'(投影坐标{x}, {y})对应的行列号是({row}, {col})') - ProjectedCoordinates = x, y # ProjectedCoordinates 投影坐标 - RowNumber = row, col # RowNumber 对应的行列号 - # 根据行列号得到地理坐标 - x, y = ds.xy(row, col) # 中心点的坐标 - print(f'行列号({row}, {col})对应的中心投影坐标是({x}, {y})') # - CenterProjectionCoordinates = x, y # CenterProjectionCoordinates 中心投影坐标 - # 'C:/Users/HP/Desktop/Number tube/GF4_PMI_E119.8_N35.3_20210908_L1A0000417337/GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml' - # 传入xml文件路径，解析xml文件 - # xml_name 存储后的xml文件的路径+文件名 - xml_dict = xml_to_dict(file_path+xml_name) - StartTime = xml_dict['ProductMetaData']['StartTime'] # 开始采集时间 - EndTime = xml_dict['ProductMetaData']['EndTime'] # 结束采集时间 - CloudPercent = xml_dict['ProductMetaData']['CloudPercent'] # 云覆盖量百分比 - TopLeftLatitude = xml_dict['ProductMetaData']['TopLeftLatitude'] # 左上纬度 - TopLeftLongitude = xml_dict['ProductMetaData']['TopLeftLongitude'] # 左上经度 - TopRightLatitude = xml_dict['ProductMetaData']['TopRightLatitude'] # 右上纬度 - TopRightLongitude = xml_dict['ProductMetaData']['TopRightLongitude'] # 右上经度 - BottomRightLatitude = xml_dict['ProductMetaData']['BottomRightLatitude'] # 右下纬度 - BottomRightLongitude = xml_dict['ProductMetaData']['BottomRightLongitude'] # 右下经度 - BottomLeftLatitude = xml_dict['ProductMetaData']['BottomLeftLatitude'] # 左下纬度 - BottomLeftLongitude = xml_dict['ProductMetaData']['BottomLeftLongitude'] # 左下经度 - boundaryGeomStr = f'POLYGON(({TopLeftLatitude} {TopLeftLongitude},' \ - f'{TopRightLatitude} {TopRightLongitude},' \ - f'{BottomRightLatitude} {BottomRightLongitude},' \ - f'{BottomLeftLatitude} {BottomLeftLongitude},' \ - f'{TopLeftLatitude} {TopLeftLongitude}))' - # ThumbnailPath 存储后的缩略图的路径+文件名 ThumbnailName 缩略图文件名称 - # xmlPath 存储后的xml文件路径+文件名 xmlFileName xml文件名称 - sensing_dict = {'StartTime': StartTime, 'EndTime': EndTime, 'CloudPercent': CloudPercent, - 'boundaryGeomStr': boundaryGeomStr, 'DataFormat': DataFormat, 'NumberBands': NumberBands, - 'ImageWidth': ImageWidth, 'ImageHeight': ImageHeight, 'GeographicScope': GeographicScope, - #'ReflectionParameter': ReflectionParameter, 'ProjectionDefinition': ProjectionDefinition, - #'FirstBindMax': FirstBindMax, 'FirstBindMin': FirstBindMin, - 'FirstBindAverage': FirstBindAverage, - 'ProjectedCoordinates': ProjectedCoordinates, 'RowNumber': RowNumber, - #'CenterProjectionCoordinates': CenterProjectionCoordinates, - 'CollectionCode': CollectionCode, - "ThumbnailPath": file_path+"GF4_IRS_E119.8_N35.3_20210908_L1A0000417337_thumb.jpg", - "ThumbnailName": "GF4_IRS_E119.8_N35.3_20210908_L1A0000417337_thumb.jpg", - "xmlPath": "", "xmlFileName": "", - 'DirectoryDepth': 'day'} - return sensing_dict - - -if __name__ == '__main__': - file_path = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_E119.8_N35.3_20210908_L1A0000417337/GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.tiff' - xml_path = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_E119.8_N35.3_20210908_L1A0000417337/GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml' - gf4_pmi_001(file_path, xml_path) diff --git a/util/scan_file_util.py b/util/scan_file_util.py deleted file mode 100644 index 4a4140b..0000000 --- a/util/scan_file_util.py +++ /dev/null @@ -1,1219 +0,0 @@ -""" -Author : XinYi Song -Time : 2021/11/3 9:00 -Desc: -""" -import os -import time - -import requests - -from application.settings import Config -from common.tools.dms import dms_login, dms_task_record, dms_sensing_data -from scan_data.GetMetaInfo import GetGFPMSData, GetGF3MDJData, GetH08Data, GetSentinel1Data, GetSentinel2Data, \ - GetSentinel3OLData, GetHJ1Data, GetZY3Data, GetSNPPData -from scan_data.example import GetJPSSData -from upload.upload_client import upload_file_client -from util.http_file_upload import upload_client -from util.http_util import httpUtil -from util.remote_sensing_util import gf4_pmi_001 -from util.xml_util import dict_to_xml -from apscheduler.schedulers.blocking import BlockingScheduler - -sched = BlockingScheduler() - - -def list_dir(file_dir): - """ - 通过 listdir 得到的是仅当前路径下的文件名，不包括子目录中的文件，如果需要得到所有文件需要递归 - """ - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = '' # 解析出的xml文件保存的路径 - ThumbnailPath = '' # 解析出的缩略图保存的路径 - for cur_file in d: - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - file_name = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.tiff' - xml_path = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml' - # 解析遥感数据文件（demo） - gf4_sensing = gf4_pmi_001(file_name, xml_path) - file_name = os.path.basename(path).split('.')[0] - xml_path = 'D:/file/work/pythonyuanma/dms_management/xml/' + file_name + '.xml' - # 将遥感数据写入xml中，并保存本地文件夹中 - dict_to_xml(gf4_sensing, xml_path) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(xml_path, 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项 - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(gf4_sensing['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项 - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = gf4_sensing['CollectionCode'] - DirectoryDepth = gf4_sensing['DirectoryDepth'] - StartTime = gf4_sensing['StartTime'] - uc = upload_client(path, DirectoryDepth, StartTime) - - StartTime = time.mktime(time.strptime(gf4_sensing['StartTime'], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(gf4_sensing['EndTime'], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": "GF4_PMI_001", "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": gf4_sensing['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": gf4_sensing['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": "GF4_PMI_001", "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_VJ102_dir(): - """ - 解析JPSS-VJ102元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\JPSS' - dir_list = os.listdir(file_dir) - print('开始扫描VJ102IMG数据集') - collectionCode = 'VJ102IMG' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'VJ102IMG' in cur_file and os.path.splitext(cur_file)[1] == '.nc': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - print(path) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = JPSSData_dict['CollectionCode'] - DirectoryDepth = JPSSData_dict['DirectoryDepth'] - StartTime = JPSSData_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_VJ103_dir(): - """ - 解析JPSS-VJ103元数据 - :param file_dir: - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\JPSS' - print('开始扫描VJ103IMG数据集') - collectionCode = 'VJ103IMG' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'VJ103IMG' in cur_file and os.path.splitext(cur_file)[1] == '.nc': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = JPSSData_dict['CollectionCode'] - DirectoryDepth = JPSSData_dict['DirectoryDepth'] - StartTime = JPSSData_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_GF1_PMS2_dir(): - """ - 获取高分 PMS卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\GF-1' - print('开始扫描GF1_PMS2_001数据集') - collectionCode = 'GF1_PMS2_001' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'GF1_PMS2' in cur_file[0:8] and os.path.splitext(cur_file)[1] == '.gz': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - GFPMS_dict = GetGFPMSData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - print(GFPMS_dict['xmlPath']) - files = {'file': open(GFPMS_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GFPMS_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GFPMS_dict['CollectionCode'] - DirectoryDepth = GFPMS_dict['DirectoryDepth'] - StartTime = GFPMS_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(GFPMS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GFPMS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GFPMS_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GFPMS_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_GF3MDJ_dir(): - """ - 获取高分3号MDJ（GF-3 MDJ）卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\GF-3' - print('开始扫描GF3_MDJ_SS数据集') - collectionCode = 'GF3_MDJ_SS' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'GF3_MDJ' in cur_file[0:7] and os.path.splitext(cur_file)[1] == '.gz': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - GF3_MDJ_SS_dict = GetGF3MDJData(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GF3_MDJ_SS_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GF3_MDJ_SS_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GF3_MDJ_SS_dict['CollectionCode'] - DirectoryDepth = GF3_MDJ_SS_dict['DirectoryDepth'] - StartTime = GF3_MDJ_SS_dict['StartTime'] - # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) - uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) - - StartTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GF3_MDJ_SS_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GF3_MDJ_SS_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_H08_dir(): - """ - 获取葵花8卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\葵花8' - print('开始扫描H08数据集') - collectionCode = 'NC_H08' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'NC_H08' in cur_file[0:6] and os.path.splitext(cur_file)[1] == '.nc': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - GetH08_dict = GetH08Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetH08_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetH08_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetH08_dict['CollectionCode'] - DirectoryDepth = GetH08_dict['DirectoryDepth'] - StartTime = GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GetH08_dict['ProduceTime'][0:10] + ' '+ GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetH08_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetH08_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_Sentinel1_dir(): - """ - 获取哨兵1号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\SENTINEL-1' - print('开始扫描Sentinel1数据集') - collectionCode = 'S1A_IW_GRDH' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'S1A_IW_GRDH' in cur_file[0:11] and os.path.splitext(cur_file)[1] == '.zip': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - Sentinel1_dict = GetSentinel1Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel1_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel1_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = Sentinel1_dict['CollectionCode'] - DirectoryDepth = Sentinel1_dict['DirectoryDepth'] - StartTime = Sentinel1_dict['ProduceTime'][0:10] + ' ' + Sentinel1_dict['ProduceTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(Sentinel1_dict['StartTime'][0:10] + ' ' + Sentinel1_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(Sentinel1_dict['StopTime'][0:10] + ' ' + Sentinel1_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": Sentinel1_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": Sentinel1_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_Sentinel2_dir(): - """ - 获取哨兵2号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\SENTINEL-2' - print('开始扫描Sentinel2数据集') - collectionCode = 'S2B' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'S2B' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.zip': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - Sentinel2_dict = GetSentinel2Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel2_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel2_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = Sentinel2_dict['CollectionCode'] - DirectoryDepth = Sentinel2_dict['DirectoryDepth'] - StartTime = Sentinel2_dict['ProduceTime'][0:10] + ' ' + Sentinel2_dict['ProduceTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(Sentinel2_dict['StartTime'][0:10] + ' ' + Sentinel2_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(Sentinel2_dict['StopTime'][0:10] + ' ' + Sentinel2_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": Sentinel2_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": Sentinel2_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_Sentinel3OL_dir(): - """ - 获取哨兵3号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\SENTINEL-3' - print('开始扫描Sentinel3数据集') - collectionCode = 'Sentinel3_OLCI' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'S3B' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.zip': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - print(path) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - Sentinel3OL_dict = GetSentinel3OLData(path, xmlPath, ThumbnailPath) - print(Sentinel3OL_dict) - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel3OL_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(Sentinel3OL_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = Sentinel3OL_dict['CollectionCode'] - DirectoryDepth = Sentinel3OL_dict['DirectoryDepth'] - StartTime = Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(Sentinel3OL_dict['StopTime'][0:10] + ' ' + Sentinel3OL_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": Sentinel3OL_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": Sentinel3OL_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_HJ1_dir(): - """ - 获取环境1号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\HJ-1' - print('开始扫描环境1号数据集') - collectionCode = 'HJ-1' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'HJ1' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.gz': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - GetHJ1Data_dict = GetHJ1Data(path, xmlPath, ThumbnailPath) - - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetHJ1Data_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetHJ1Data_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetHJ1Data_dict['CollectionCode'] - DirectoryDepth = GetHJ1Data_dict['DirectoryDepth'] - StartTime = GetHJ1Data_dict['ProductTime'][0:10] + ' ' + GetHJ1Data_dict['ProductTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(GetHJ1Data_dict['StartTime'][0:10] + ' ' + GetHJ1Data_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GetHJ1Data_dict['EndTime'][0:10] + ' ' + GetHJ1Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetHJ1Data_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetHJ1Data_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_ZY3_dir(): - """ - 获取资源3号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\ZY-3' - print('开始扫描资源3号数据集') - collectionCode = 'ZY-3' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'ZY3' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.gz': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - GetZY3Data_dict = GetZY3Data(path, xmlPath, ThumbnailPath) - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetZY3Data_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetZY3Data_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetZY3Data_dict['CollectionCode'] - DirectoryDepth = GetZY3Data_dict['DirectoryDepth'] - StartTime = GetZY3Data_dict['ProduceTime'][0:10] + ' ' + GetZY3Data_dict['ProduceTime'][11:19] - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(GetZY3Data_dict['StartTime'][0:10] + ' ' + GetZY3Data_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GetZY3Data_dict['EndTime'][0:10] + ' ' + GetZY3Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetZY3Data_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetZY3Data_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -def scan_SNPP_dir(): - """ - 获取资源3号卫星元数据 - :return: - """ - # file_dir = 'E:/数管' - file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\VIIRS' - print('开始扫描VNP02IMG数据集') - collectionCode = 'VNP02IMG' - # 用户登录 - token_s = dms_login() - # 判断定时任务是否在进行 - task = dms_task_record(token_s, collectionCode) - # 如果不是空说明正在进行 - if task is not None and len(task) > 0: - return - fileNameList = [] - dms_list = dms_sensing_data(token_s, collectionCode) - for dms in dms_list: - fileNameList.append(dms['fileName']) - dir_list = os.listdir(file_dir) - # 判断扫描出的文件和已有的文件，将多出的文件进行解析 - d = [y for y in dir_list if y not in fileNameList] - if d is None or len(d) == 0: - print('没有多余的遥感数据文件，终止程序') - return - file_total_size = "" - file_total_name = "" - xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 - ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 - for cur_file in d: - if 'VNP02IMG' in cur_file[0:8] and os.path.splitext(cur_file)[1] == '.nc': - - # 获取文件的绝对路径 - path = os.path.join(file_dir, cur_file) - if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 - # 解析遥感数据文件（demo） - GetSNPPData_dict = GetSNPPData(path, xmlPath, ThumbnailPath) - print(GetSNPPData_dict) - # 配置文件服务器参数 - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetSNPPData_dict['xmlPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - xml = requests.post(url, data=options, files=files) - - url = Config.DFS_UPLOAD_URL - files = {'file': open(GetSNPPData_dict['ThumbnailPath'], 'rb')} - options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} - # 上传生成的xml文件到文件服务器 - ThumbnailName = requests.post(url, data=options, files=files) - - CollectionCode = GetSNPPData_dict['CollectionCode'] - DirectoryDepth = GetSNPPData_dict['DirectoryDepth'] - StartTime = GetSNPPData_dict['ProductionTime'][0:10] + ' ' + GetSNPPData_dict['ProductionTime'][11:19] - # uc = upload_client(path, DirectoryDepth, StartTime, collectionCode) - uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) - - StartTime = time.mktime(time.strptime(GetSNPPData_dict['StartTime'][0:10] + ' ' + GetSNPPData_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) - EndTime = time.mktime(time.strptime(GetSNPPData_dict['EndTime'][0:10] + ' ' + GetSNPPData_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) - - # 入库遥感数据 - res_data = httpUtil(url=Config.RESING_DATA_URL, - data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, - "shootingTimeEndTs": EndTime, - "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], - "fileSize": uc['file_size'], "cloudCoverage": GetSNPPData_dict['CloudPercent'], - "metaInformationFile": xml.json()['path'], - "thumbnailFile": ThumbnailName.json()['path'], - "remarks": "", "boundaryGeomStr": GetSNPPData_dict['boundaryGeomStr']}, - token=token_s).post_no_patam_herder() - print(res_data.json()['data']) - - file_total_size = file_total_size + str(uc['file_size']) - file_total_size = file_total_size + "," - - file_total_name = file_total_name + uc['fileName'] - file_total_name = file_total_name + "," - # print("========"+suffix) - print("{0} : is file!".format(cur_file)) - if os.path.isdir(path): - print("{0} : is dir!".format(cur_file)) - list_dir(path) # 递归子目录 - if uc['type'] == 'ok': - continue - # 添加遥感数据归档任务 - res = httpUtil(url=Config.DATA_TASK_URL, - data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, - "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() - task_code = res.json()['data'] - - # 结束遥感数据归档任务 - header = {"Authorization": token_s} - res = requests.post(url=Config.DATA_END_TASK_URL, - - params={"taskCode": task_code}, headers=header).json() - - -if __name__ == '__main__': - # file_dir = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_001/sensingdata' - # list_dir(file_dir) - # file_dir = 'E:/数管' - # - scan_VJ102_dir() - # - # scan_VJ103_dir() - - # scan_GF1_PMS2_dir() - # scan_GF3MDJ_dir() - # scan_H08_dir() - # scan_Sentinel1_dir() - # scan_Sentinel2_dir() - # scan_Sentinel3OL_dir() - # scan_HJ1_dir() - # scan_ZY3_dir() - # scan_SNPP_dir() \ No newline at end of file diff --git a/util/simple_sqlite3_tool.py b/util/simple_sqlite3_tool.py index 75d85c5..bd32962 100644 --- a/util/simple_sqlite3_tool.py +++ b/util/simple_sqlite3_tool.py @@ -1,6 +1,5 @@ # coding: utf-8 -# Author：boxker -# Mail：icjb@foxmail.com +# Author：tajochen import sqlite3 import os @@ -13,79 +12,84 @@ class SimpleSQLite3Tool: 编写这个类主要是为了封装sqlite，继承此类复用方法 """ - def __init__(self, filename="stsql"): - """ - 初始化数据库，默认文件名 stsql.db - filename：文件名 - """ - self.filename = filename - self.db = sqlite3.connect(self.filename) - self.c = self.db.cursor() - def close(self): - """ - 关闭数据库 - """ - self.c.close() - self.db.close() +def __init__(self, filename="stsql"): + """ + 初始化数据库，默认文件名 stsql.db + filename：文件名 + """ + self.filename = filename + self.db = sqlite3.connect(self.filename) + self.c = self.db.cursor() - def execute(self, sql, param=None): - """ - 执行数据库的增、删、改 - sql：sql语句 - param：数据，可以是list或tuple，亦可是None - retutn：成功返回True - """ - try: - if param is None: - self.c.execute(sql) - else: - if type(param) is list: - self.c.executemany(sql, param) - else: - self.c.execute(sql, param) - count = self.db.total_changes - self.db.commit() - except Exception as e: - print(e) - return False, e - if count > 0: - return True - else: - return False - def query(self, sql, param=None): - """ - 查询语句 - sql：sql语句 - param：参数,可为None - retutn：成功返回True - """ +def close(self): + """ + 关闭数据库 + """ + self.c.close() + self.db.close() + + +def execute(self, sql, param=None): + """ + 执行数据库的增、删、改 + sql：sql语句 + param：数据，可以是list或tuple，亦可是None + return：成功返回True + """ + try: if param is None: self.c.execute(sql) else: - self.c.execute(sql, param) - return self.c.fetchall() + if type(param) is list: + self.c.executemany(sql, param) + else: + self.c.execute(sql, param) + count = self.db.total_changes + self.db.commit() + except Exception as e: + print(e) + return False, e + if count > 0: + return True + else: + return False - # def set(self,table,field=" * ",where="",isWhere=False): - # self.table = table - # self.filed = field - # if where != "" : - # self.where = where - # self.isWhere = True - # return True + +def query(self, sql, param=None): + """ + 查询语句 + sql：sql语句 + param：参数,可为None + return：成功返回True + """ + if param is None: + self.c.execute(sql) + else: + self.c.execute(sql, param) + return self.c.fetchall() + + +# def set(self,table,field=" * ",where="",isWhere=False): +# self.table = table +# self.filed = field +# if where != "" : +# self.where = where +# self.isWhere = True +# return True if __name__ == "__main__": # 数据库文件位置 - sql = SimpleSQLite3Tool("../dms_client.db") - # f = sql.execute("create table test (id int not null,name text not null,age int);") - # print("ok") - # sql.execute("insert into test (id,name,age) values (?,?,?);", [(1, 'abc', 15), (2, 'bca', 16)]) - # res = sql.query("select * from test;") - # print(res) - # sql.execute("insert into test (id,name) values (?,?);", (3, 'bac')) - # res = sql.query("select * from test where id=?;", (3,)) + sql = SimpleSQLite3Tool("../test.db") + f = sql.execute("create table test (id int not null,name text not null,age int);") + print("ok") + sql.execute("insert into test (id,name,age) values (?,?,?);", [(1, 'abc', 15), (2, 'bca', 16)]) + res = sql.query("select * from test;") + print(res) + sql.execute("insert into test (id,name) values (?,?);", (3, 'bac')) + res = sql.query("select * from test where id=?;", (3,)) res = sql.query("select * from data_collection_info;") print(res) sql.close() diff --git a/upload/upload_client.py b/util/upload_client.py similarity index 69% rename from upload/upload_client.py rename to util/upload_client.py index 72b2de1..f14f6ec 100644 --- a/upload/upload_client.py +++ b/util/upload_client.py @@ -8,8 +8,7 @@ import sys import re import os from hashlib import md5 - -from util.file_store_path import file_store_path_year, file_store_path_month, file_store_path_day +import time FILE_DIR = os.path.dirname(__file__) @@ -26,6 +25,27 @@ def bar(num=1, sum=100): sys.stdout.write(temp) +def file_store_path(time_stamp): + """ + :param time_stamp: 时间戳类型时间 + :return: + """ + now = int(round(time_stamp * 1000)) + t = time.localtime(now / 1000) + return os.path.join('E:/data/upload', str(t[0]), str(t[1]), str(t[2])) + + +def file_store_path_year(data_str_time, upload_path, conllection_code): + """ + 目录到年 + :param upload_path: + :param data_str_time: 字符串类型 + :return: + """ + t = time.strptime(data_str_time, '%Y-%m-%d %H:%M:%S') + return os.path.join(upload_path, conllection_code, str(t[0])) + + def md5_file(name): m = md5() a_file = open(name, 'rb') # 需要使用二进制格式读取文件内容 @@ -34,6 +54,28 @@ def md5_file(name): return m.hexdigest() +def file_store_path_month(data_str_time, upload_path, conllection_code): + """ + 目录到月 + :param upload_path: + :param data_str_time: + :return: + """ + t = time.strptime(data_str_time, '%Y-%m-%d %H:%M:%S') + return os.path.join(upload_path, conllection_code, str(t[0]), str(t[1])) + + +def file_store_path_day(data_str_time, upload_path, conllection_code): + """ + 目录到日 + :param upload_path: + :param data_str_time: 字符串类型的时间 + :return: + """ + t = time.strptime(data_str_time, '%Y-%m-%d %H:%M:%S') + return os.path.join(upload_path, conllection_code, str(t[0]), str(t[1]), str(t[2])) + + def upload_file_client(file_path, depth, dateTime, collectionCode): ck = socket.socket() ck.connect(('192.168.2.9', 9002)) diff --git a/util/zxby.py b/util/zxby.py deleted file mode 100644 index 261eaf7..0000000 --- a/util/zxby.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Author : XinYi Song -Time : 2021/10/9 9:43 -Desc: -""" -import os, sys, subprocess, tempfile, time - -# 创建临时文件夹,返回临时文件夹路径 -TempFile = tempfile.mkdtemp(suffix='_test', prefix='python_') -# 文件名 -FileNum = int(time.time() * 1000) -# python编译器位置 -EXEC = sys.executable - - -# 获取python版本 -def get_version(): - v = sys.version_info - version = "python %s.%s" % (v.major, v.minor) - return version - - -# 获得py文件名 -def get_pyname(): - global FileNum - return 'test_%d' % FileNum - - -# 接收代码写入文件 -def write_file(pyname, code): - fpath = os.path.join(TempFile, '%s.py' % pyname) - with open(fpath, 'w', encoding='utf-8') as f: - f.write(code) - print('file path: %s' % fpath) - return fpath - - -# 编码 -def decode(s): - try: - return s.decode('utf-8') - except UnicodeDecodeError: - return s.decode('gbk') - - # 主执行函数 - - -def main(code): - r = dict() - r["version"] = get_version() - pyname = get_pyname() - fpath = write_file(pyname, code) - try: - # subprocess.check_output 是 父进程等待子进程完成，返回子进程向标准输出的输出结果 - # stderr是标准输出的类型 - outdata = decode(subprocess.check_output([EXEC, fpath], stderr=subprocess.STDOUT, timeout=5)) - except subprocess.CalledProcessError as e: - # e.output是错误信息标准输出 - # 错误返回的数据 - r["code"] = 'Error' - r["output"] = decode(e.output) - return r - else: - # 成功返回的数据 - r['output'] = outdata - return r - finally: - # 删除文件(其实不用删除临时文件会自动删除) - try: - os.remove(fpath) - except Exception as e: - exit(1) - - -# if __name__ == '__main__': -# code = "print(11);print(22)" -# print(main(code)) -