#!/usr/bin/env python3 # -*- coding:utf-8 -*- import os import requests import json import time import hashlib import subprocess import math import redis from datetime import datetime class PddPublishClient: appType = '' redisCache = '' def __init__(self, appType): self.appType = appType self.redisCache = redis.Redis(host='r-2zeelkx0uxgwn845lzpd.redis.rds.aliyuncs.com',port=6379,db=50,password='IR(@r2*MFASrm23$r3*') try: self.redisCache.ping() except redis.exceptions.ConnectionError: print("Failed to connect to Redis") exit() pass def getUserInfo(self): appTypeAndUserInfoMap = { 'ms': { 'mobile': '', 'subAccountName': 'publish@1544', 'password': 'vsd4sfhskf@34' }, 'jb': { 'mobile': '', 'subAccountName': 'publish@279', 'password': 'lds2fbx@fvxwW@faS' }, 'tbk': { 'mobile': '', 'subAccountName': 'publish@114587', 'password': 'ldsfvxwW@%faS' }, } return appTypeAndUserInfoMap[self.appType] def getAppInfo(self, appName, pubEnv): appNameAndPubEnvAndAppInfoMap = { # v2 'mspdd-v2': { 'pre': { 'name': '妙手上货-V2-预发布', 'appInstanceIds': ['app-685dd0de'], }, 'pub': { 'name': '妙手上货-V2-正式', 'appInstanceIds': ['app-2dd0b366'], } }, 'mspddsp-v2': { 'pre': { 'name': '妙手商品-V2-预发布', 'appInstanceIds': ['app-d5ad844a'], }, 'pub': { 'name': '妙手商品-V2-正式', 'appInstanceIds': ['app-aafd8833'], } }, 'mspddodsh-v2': { 'pre': { 'name': '妙手采购-上货V2-预发布', 'appInstanceIds': ['app-4e3c6b6c'], }, 'pub': { 'name': '妙手采购-上货V2-正式', 'appInstanceIds': ['app-25d1b00a'], } }, 'pddbksh-v2': { 'pre': { 'name': '百客上货-V2-预发布', 'appInstanceIds': ['app-f6b3f2ef'], }, 'pub': { 'name': '百客上货-V2-正式', 'appInstanceIds': ['app-bf955b95'], } }, 'pddry-v2': { 'pre': { 'name': 'ruby-V2-预发布', 'appInstanceIds': ['app-f1760922'], }, 'pub': { 'name': 'ruby-V2-正式', 'appInstanceIds': ['app-c2e828b2'], } }, # od-v2 'mspddod-v2': { 'pre': { 'name': '妙手采购-V2-预发布', 'appInstanceIds': ['app-ece23399'], }, 'pub': { 'name': '妙手采购-V2-正式', 'appInstanceIds': ['app-393e7bb7', 'app-c55a76ff'], } }, 'mspddop-v2': { 'pre': { 'name': '妙手打单-V2-预发布', 'appInstanceIds': ['app-4d32de6d'], }, 'pub': { 'name': '妙手打单-V2-正式', 'appInstanceIds': ['app-ae9c5108'], } }, 'pddryop-v2': { 'pre': { 'name': '聚宝打单-V2-预发布', 'appInstanceIds': ['app-b0c0f8e0'], }, 'pub': { 'name': '聚宝打单-V2-正式', 'appInstanceIds': ['app-24e33e07'], } }, 'pddry-opv2': { 'pre': { 'name': 'ruby-OPV2-预发布', 'appInstanceIds': ['app-f12e1266'], }, 'pub': { 'name': 'ruby-OPV2-正式', 'appInstanceIds': ['app-5764f04f', 'app-70872bd0'], } }, # od-mobile 'mspddod-mobile': { 'pre': { 'name': '妙手下单-V2-预发布', 'appInstanceIds': ['app-875e1274'], }, 'pub': { 'name': '妙手下单-V2-正式', 'appInstanceIds': ['app-bf027d32'], } }, # mssp + mobile 'mspddsp': { 'pre': { 'name': '妙手商品-预发布', 'appInstanceIds': ['app-47ad6107'], }, 'pub': { 'name': '妙手商品-正式', 'appInstanceIds': ['app-f344e7ea', 'app-b6daaf65'], } }, # erp 'mspddod': { 'pre': { 'name': '妙手采购-预发布', 'appInstanceIds': ['app-8e712ae8'], }, 'pub': { 'name': '妙手采购-正式', 'appInstanceIds': ['app-1188caec', 'app-1283dc71'], } }, 'pddry': { 'pre': { 'name': '聚宝订单助手-预发布', 'appInstanceIds': ['app-402b17b5'], }, 'pub': { 'name': '聚宝订单助手-正式', 'appInstanceIds': ['app-f4094dd3', 'app-34e4b655'], } }, # goods 'mspdd': { 'pre': { 'name': '妙手上货-预发布', 'appInstanceIds': ['app-1c10554b'], }, 'pub': { 'name': '妙手上货-正式', 'appInstanceIds': ['app-5826c7f4', 'app-bf150398'], } }, 'pddbksh': { 'pre': { 'name': '百客上货-预发布', 'appInstanceIds': ['app-de1f4d81'], }, 'pub': { 'name': '百客上货-正式', 'appInstanceIds': ['app-c3ad42b0', 'app-79827402'], } }, # order 'mspddop': { 'pre': { 'name': '妙手打单-预发布', 'appInstanceIds': ['app-9fca3769'], }, 'pub': { 'name': '妙手打单-正式', 'appInstanceIds': ['app-3f67858a', 'app-c323ef63'], } }, 'mspddopdzmd': { 'pre': { 'name': '妙手打单电子面单-预发布', 'appInstanceIds': ['app-229676fc'], }, 'pub': { 'name': '妙手打单电子面单-正式', 'appInstanceIds': ['app-7f6dcbf3', 'app-09d99c15'], } }, 'pddryop': { 'pre': { 'name': '聚宝打单-预发布', 'appInstanceIds': ['app-d6637fca'], }, 'pub': { 'name': '聚宝打单-正式', 'appInstanceIds': ['app-89010d06'], } }, 'pddTaoBaiKe': { 'pre': { # 'name': '淘百客助手-预发布', # 'appInstanceIds': ['app-6f7f90d8'], }, 'pub': { 'name': '淘百客助手-正式', 'appInstanceIds': ['app-fd7e670f'], } }, } return appNameAndPubEnvAndAppInfoMap[appName][pubEnv] def publishSingle(self, appName, pubEnv, zipFilePath): appInfo = self.getAppInfo(appName, pubEnv) if not appInfo: return True appChineseName = appInfo['name'] appInstanceIds = appInfo['appInstanceIds'] cosKey = self.uploadFile(appName, zipFilePath) for appInstanceId in appInstanceIds: print('start publish %s[%s]' % (appChineseName, appInstanceId)) buildId = self.createProgram(appName, appInstanceId, cosKey, pubEnv) deployId = self.startDeploy(appName, appInstanceId, buildId) print('finish publish %s[%s] deployId[%s]' % (appChineseName, appInstanceId, deployId)) if not deployId: return False return True def tryGetAccessTokenLock(self, redisKey): tryTimes = 0 while tryTimes < 100: tryTimes += 1 ret = self.redisCache.set(redisKey, 1, nx=True, ex=10) if ret: # print('lockKey[%s] lock success' % redisKey) return True print('appType[%s] tryGetAccessTokenLock tryTimes[%s]' % (self.appType, tryTimes)) time.sleep(0.1) continue return False def delAccessTokenLock(self, redisKey): ret = self.redisCache.delete(redisKey) print('delAccessTokenLock[%s] ret:%s' % (redisKey, ret)) def getAccessTokenRedisKey(self): now = datetime.now() return 'pdd_publish_atk_%s_%s' % (self.appType, now.strftime('%Y%m%d')) def getAccessToken(self): lockKey = 'pdd_publish_atk_%s_lock' % self.appType isSuccess = self.tryGetAccessTokenLock(lockKey) if isSuccess is False: print('appType[%s] try get accessToken lock fail' % self.appType) exit() userInfo = self.getUserInfo() redisKey = self.getAccessTokenRedisKey() accessToken = self.redisCache.get(redisKey) if accessToken: print('get accessToken from cache') self.delAccessTokenLock(lockKey) return accessToken.decode('utf-8') url = "https://open-api.pinduoduo.com/ci/user/login" payload = json.dumps({ "mobile": userInfo['mobile'], "subAccountName": userInfo['subAccountName'], "password": userInfo['password'] }) headers = { 'Content-Type': 'application/json', } response = requests.request("POST", url, headers=headers, data=payload) resultObj = json.loads(response.text) if 'result' not in resultObj or 'rememberToken' not in resultObj['result']: print('%s账号：%s 手机号：%s 登录失败 原因：%s' % (self.appType, userInfo['subAccountName'], userInfo['mobile'], resultObj['errorMsg'])) self.redisCache.delete(redisKey) self.delAccessTokenLock(lockKey) exit() accessToken = resultObj['result']['rememberToken'] print('%s账号：%s 手机号：%s 登录成功 accessToken：%s' % (self.appType, userInfo['subAccountName'], userInfo['mobile'], accessToken)) self.redisCache.set(redisKey, accessToken, ex=86400) self.delAccessTokenLock(lockKey) return accessToken def uploadFile(self, appName, filePath): fileSize = os.path.getsize(filePath) maxFileSize = 10 * 1024 * 1024 if fileSize <= maxFileSize: return self.uploadSmallFile(appName, filePath) return self.uploadFileByPart(appName, filePath) def uploadFileByPart(self, appName, filePath): dirName = os.path.dirname(filePath) partFilesDirName = dirName + '/part_files_tmp/' + appName fileName = os.path.basename(filePath) fileNameWithoutExt = os.path.splitext(fileName)[0] os.system('rm -rf %s && mkdir -p %s' % (partFilesDirName, partFilesDirName)) partSize = 1024 * 1024 * 10 self.splitFile(filePath, partFilesDirName, partSize) fileSize = os.path.getsize(filePath) needPartCount = math.ceil(fileSize / partSize) cmd = 'ls -l %s | grep "%s" | wc -l ' % (partFilesDirName, fileNameWithoutExt) output = self.execCommand(cmd) output = output.strip() print('get filePartCountCmd[%s] filePath[%s] needPartCount[%s] output[%s]' % (cmd, filePath, needPartCount, output)) if not output: print('appName[%s] filePath[%s] partFilesDirName[%s] get file part count fail' % (appName, filePath, partFilesDirName)) exit() filePartCount = int(output) if filePartCount != needPartCount: print('appName[%s] fileName[%s] needPartCount[%s] filePartCount[%s]' % (appName, fileName, needPartCount, filePartCount)) exit() (uploadId, cosKey) = self.uploadFilePartInit(appName, filePath) for partNumber in range(1, filePartCount + 1): partFilePath = partFilesDirName + ('/%s_%s.zip' % (fileNameWithoutExt, partNumber)) self.uploadPartFile(appName, partNumber, partFilePath, uploadId, cosKey) cosKey = self.uploadFileComplete(appName, uploadId, cosKey) return cosKey def splitFile(self, filePath, partFilesDirName, partSize = 1024 * 1024 * 10): if not os.path.isfile(filePath): print('%s 文件不存在' % (filePath)) exit() fileName = os.path.basename(filePath) fileNameWithoutExt = os.path.splitext(fileName)[0] with open(filePath, "rb") as reader: partNumber = 1 while True: partContent = reader.read(partSize) if not partContent: print("split done.") break partFilePath = partFilesDirName + ('/%s_%s.zip' % (fileNameWithoutExt, partNumber)) with open(partFilePath,"wb") as writer: writer.write(partContent) partNumber += 1 def uploadFileComplete(self, appName, uploadId, cosKey): accessToken = self.getAccessToken() print('appName[%s] uploadId[%s] cosKey[%s] accessToken[%s] start uploadFileComplete' % (appName, uploadId, cosKey, accessToken)) url = "https://open-api.pinduoduo.com/ci/cloud/file/upload/part/complete" payload = json.dumps({ "uploadId": uploadId, "cosKey": cosKey, }) headers = { 'content-type': 'application/json;charset=UTF-8', 'ci-access-token': accessToken, } response = requests.request("POST", url, headers=headers, data=payload) resultObj = json.loads(response.text) if resultObj['errorMsg']: self.checkAndProcessInvalidAccessToken(appName, resultObj) print('appName[%s] uploadId[%s] cosKey[%s] uploadFileComplete fail: %s' % (appName, uploadId, cosKey, resultObj['errorMsg'])) exit() cosKey = resultObj['result']['cosKey'] print('appName[%s] cosKey[%s] end uploadFileComplete' % (appName, cosKey)) return cosKey def uploadFilePartInit(self, appName, filePath): accessToken = self.getAccessToken() print('appName[%s] start uploadFilePartInit filePath[%s] accessToken[%s]' % (appName, filePath, accessToken)) fileName = os.path.basename(filePath) url = "https://open-api.pinduoduo.com/ci/cloud/file/upload/part/init" payload = json.dumps({ "fileName": fileName, }) headers = { 'content-type': 'application/json;charset=UTF-8', 'ci-access-token': accessToken, } response = requests.request("POST", url, headers=headers, data=payload) resultObj = json.loads(response.text) if resultObj['errorMsg']: self.checkAndProcessInvalidAccessToken(appName, resultObj) print('appName[%s] uploadFilePartInit fail: %s' % (appName, resultObj['errorMsg'])) exit() uploadId = resultObj['result']['uploadId'] cosKey = resultObj['result']['cosKey'] print('appName[%s] end uploadFilePartInit fileName[%s] uploadId[%s] cosKey[%s]' % (appName, fileName, uploadId, cosKey)) return (uploadId, cosKey) def getFileMD5(self, filePath): if os.path.isfile(filePath): md5obj = hashlib.md5() maxbuf = 8192 f = open(filePath,'rb') while True: buf = f.read(maxbuf) if not buf: break md5obj.update(buf) f.close() hash = md5obj.hexdigest() return str(hash).upper() return None def uploadPartFile(self, appName, partNumber, filePath, uploadId, cosKey): accessToken = self.getAccessToken() print('appName[%s] start uploadPartFile filePath[%s] accessToken[%s]' % (appName, filePath, accessToken)) fileName = os.path.basename(filePath) fileSize = os.path.getsize(filePath) partMD5 = self.getFileMD5(filePath) url = "https://open-api.pinduoduo.com/ci/cloud/file/upload/part" payload={ 'uploadId': uploadId, 'partNumber': partNumber, 'partSize': fileSize, 'cosKey': cosKey, 'partMD5': partMD5 } files=[ ('file',(fileName,open(filePath,'rb'),'application/zip')) ] headers = { 'ci-access-token': accessToken, } response = requests.request("POST", url, headers=headers, data=payload, files=files) resultObj = json.loads(response.text) if resultObj['errorMsg']: self.checkAndProcessInvalidAccessToken(appName, resultObj) print('appName[%s] uploadPartFile fail: %s' % (appName, resultObj['errorMsg'])) exit() print('appName[%s] end uploadPartFile fileName[%s] cosKey[%s]' % (appName, fileName, cosKey)) def uploadSmallFile(self, appName, filePath): accessToken = self.getAccessToken() print('appName[%s] start uploadSmallFile filePath[%s] accessToken[%s]' % (appName, filePath, accessToken)) fileName = os.path.basename(filePath) url = "https://open-api.pinduoduo.com/ci/cloud/file/upload" payload={} files=[ ('file',(fileName,open(filePath,'rb'),'application/zip')) ] headers = { 'ci-access-token': accessToken, } response = requests.request("POST", url, headers=headers, data=payload, files=files) resultObj = json.loads(response.text) if resultObj['errorMsg']: self.checkAndProcessInvalidAccessToken(appName, resultObj) print('appName[%s] uploadSmallFile fail: %s' % (appName, resultObj['errorMsg'])) exit() cosKey = resultObj['result'] print('appName[%s] end upload fileName[%s] cosKey[%s]' % (appName, fileName, cosKey)) return cosKey def createProgram(self, appName, appInstanceId, cosKey, pubEnv): accessToken = self.getAccessToken() print('appName[%s] appInstanceId[%s] cosKey[%s] accessToken[%s] start createProgram ' % (appName, appInstanceId, cosKey, accessToken)) fileName = os.path.basename(cosKey) tag = time.strftime("v%Y%m%d.%H.%M.%S") description = 'pub master' if pubEnv == 'pub' else 'pub pre' payload = json.dumps({ "appInstanceId": appInstanceId, "cosKey": cosKey, "tag": tag, "fileName": fileName, "description": description, }) headers = { 'content-type': 'application/json;charset=UTF-8', 'ci-access-token': accessToken, } url = "https://open-api.pinduoduo.com/ci/res/program/create" response = requests.request("POST", url, headers=headers, data=payload) resultObj = json.loads(response.text) if resultObj['errorMsg']: self.checkAndProcessInvalidAccessToken(appName, resultObj) self.printErrorMsgAndExit(appName, appInstanceId, '开始构建', resultObj['errorMsg']) buildId = resultObj['result']['buildId'] if self.checkBuildSuccess(appName, appInstanceId, buildId) is not True: print('%s buildId[%s] 打包超时' % (appInstanceId, buildId)) exit() print('appName[%s] appInstanceId[%s] cosKey[%s] end createProgram buildId[%s] ' % (appName, appInstanceId, cosKey, buildId)) return buildId def checkBuildSuccess(self, appName, appInstanceId, buildId): i = 1 while i < 100: accessToken = self.getAccessToken() url = "https://open-api.pinduoduo.com/ci/image/build/info" payload = json.dumps({ "buildId": buildId }) headers = { 'ci-access-token': accessToken, 'Content-Type': 'application/json', } response = requests.request("POST", url, headers=headers, data=payload) resultObj = json.loads(response.text) if resultObj['errorMsg']: self.checkAndProcessInvalidAccessToken(appName, resultObj) self.printErrorMsgAndExit(appName, appInstanceId, '检测构建状态', resultObj['errorMsg']) status = resultObj['result']['status'] if status == 3: print('appName[%s] appInstanceId[%s] buildId[%s] accessToken[%s] 打包完成' % (appName, appInstanceId, buildId, accessToken)) return True print('appName[%s] appInstanceId[%s] buildId[%s] accessToken[%s] 打包未完成 status[%s] try[%s]' % (appName, appInstanceId, buildId, accessToken, status, i)) i += 1 time.sleep(5) return False def startDeploy(self, appName, appInstanceId, buildId, rollback = False): accessToken = self.getAccessToken() print('appName[%s] appInstanceId[%s] buildId[%s] accessToken[%s] start startDeploy ' % (appName, appInstanceId, buildId, accessToken)) payload = json.dumps({ "buildId": buildId, "rollback": rollback, }) headers = { 'content-type': 'application/json;charset=UTF-8', 'ci-access-token': accessToken, } url = "https://open-api.pinduoduo.com/ci/app/deploy/start" response = requests.request("POST", url, headers=headers, data=payload) resultObj = json.loads(response.text) if resultObj['errorMsg']: self.checkAndProcessInvalidAccessToken(appName, resultObj) self.printErrorMsgAndExit(appName, appInstanceId, '开始发布', resultObj['errorMsg']) deployId = resultObj['result']['deployId'] # 因耗时久去除校验发布状态 # if self.checkDeploySuccess(appName, appInstanceId, deployId) is not True: # print('%s[%s] deployId[%s] 检测发布状态超时' % (appName, appInstanceId, deployId)) # exit() print('appName[%s] appInstanceId[%s] buildId[%s] finish deploy deployId[%s]' % (appName, appInstanceId, buildId, deployId)) return deployId def checkDeploySuccess(self, appName, appInstanceId, deployId): i = 1 while i < 100: accessToken = self.getAccessToken() url = "https://open-api.pinduoduo.com/ci/app/deploy/info" payload = json.dumps({ "deployId": deployId }) headers = { 'ci-access-token': accessToken, 'Content-Type': 'application/json', } response = requests.request("POST", url, headers=headers, data=payload) resultObj = json.loads(response.text) if resultObj['errorMsg']: self.checkAndProcessInvalidAccessToken(appName, resultObj) self.printErrorMsgAndExit(appName, appInstanceId, '检测发布状态', resultObj['errorMsg']) status = resultObj['result']['status'] if status == 3: print('appName[%s] appInstanceId[%s] deployId[%s] accessToken[%s] 发布完成' % (appName, appInstanceId, deployId, accessToken)) return True print('appName[%s] appInstanceId[%s] deployId[%s] accessToken[%s] 发布未完成 status[%s] try[%s]' % (appName, appInstanceId, deployId, accessToken, status, i)) i += 1 time.sleep(5) return False def getAppConfigList(self, appId): accessToken = self.getAccessToken() print('appId[%s] accessToken[%s] getAppConfigList ' % (appId, accessToken)) payload = json.dumps({ "appId": appId, }) headers = { 'content-type': 'application/json;charset=UTF-8', 'ci-access-token': accessToken, } url = "https://open-api.pinduoduo.com/ci/app/config/file/list" response = requests.request("POST", url, headers=headers, data=payload) resultObj = json.loads(response.text) if resultObj['errorMsg']: self.checkAndProcessInvalidAccessToken('', resultObj) self.printErrorMsgAndExit('', appId, '获取应用配置列表', resultObj['errorMsg']) configList = resultObj['result'] return configList def getAppConfigFileById(self, configFileId): accessToken = self.getAccessToken() print('configFileId[%s] accessToken[%s] getAppConfigFileById ' % (configFileId, accessToken)) payload = json.dumps({ "configFileId": configFileId, }) headers = { 'content-type': 'application/json;charset=UTF-8', 'ci-access-token': accessToken, } url = "https://open-api.pinduoduo.com/ci/app/config/file/get" response = requests.request("POST", url, headers=headers, data=payload) resultObj = json.loads(response.text) if resultObj['errorMsg']: self.checkAndProcessInvalidAccessToken('', resultObj) self.printErrorMsgAndExit('', configFileId, '获取单个应用配置', resultObj['errorMsg']) configInfo = resultObj['result'] return configInfo def updateAppConfig(self, configFileId, fileName, mountPath, content, version): accessToken = self.getAccessToken() print('configFileId[%s] fileName[%s] mountPath[%s] version[%s] accessToken[%s] updateAppConfig ' % (configFileId, fileName, mountPath, version, accessToken)) payload = json.dumps({ "configFileId": configFileId, "fileName": fileName, "mountPath": mountPath, "content": content, "version": version, }) headers = { 'content-type': 'application/json;charset=UTF-8', 'ci-access-token': accessToken, } url = "https://open-api.pinduoduo.com/ci/app/config/file/update" response = requests.request("POST", url, headers=headers, data=payload) resultObj = json.loads(response.text) if resultObj['errorMsg']: self.checkAndProcessInvalidAccessToken('', resultObj) self.printErrorMsgAndExit('', configFileId, '更新应用配置', resultObj['errorMsg']) result = resultObj['result'] return result def getFileContent(self, filePath): content = '' with open(filePath, 'r', encoding='utf-8') as file: # 读取文件的全部内容 content = file.read() return content def execCommand(self, cmd): process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) stdout, stderr = process.communicate() if stderr: print('execCommand[%s] fail: %s' % (cmd, stderr.strip())) exit() output = stdout.strip() if not output: print('execCommand[%s] fail: %s' % (cmd, 'no output')) exit() return output def printErrorMsgAndExit(self, appName, appInstanceId, stepName, errorMsg): print('应用[%s] 实例ID[%s] %s失败 原因：%s' % (appName, appInstanceId, stepName, errorMsg)) exit() def checkAndProcessInvalidAccessToken(self, appName, resultObj): if resultObj['errorCode'] != 40001: return self.redisCache.delete(self.getAccessTokenRedisKey()) print('%s账号 应用[%s] accessToken失效，请重新执行命令 原因：%s' % (self.appType, appName, resultObj['errorMsg'])) exit() # client = PddPublishClient('ms') # client.publishSingle('mspdd-v2', 'pre', '/Users/tangjianhui/Documents/ddc_zip_v2/v2/pdd-v2_pre_20230130-1142.zip')