conda的基本指令

【研究】一些QA

  返回  

利用Python遍历odps元数据信息

2021/7/20 16:29:03 浏览:

bm:表名

fq:分区

cjsj:分区创建时间

smzq:生命周期

dx:分区大小

from odps import ODPS
odps = ODPS('xxxxxxxxx', 'xxxxxxxxxxxx', 'dyx',
            endpoint='http://xxxxxxxxx.odps.ops.yun.ga/api')
odps1 = ODPS('xxxxxxxxxxxx', 'xxxxxxxxxxxxxx', 'dwd__dev',
            endpoint='http://xxxxxxxxx1.odps.ops.yun.ga/api')            
import json
project = odps.get_project()
print(project)
#获取需要执行的表
sql='''select replace(trim(mbbm),'dwd__dev.','')mbbm from t_julong_bzc_sjgxzq where xh >=60 and xh<70 and facjsj<>''  '''
#读出表结果数据
with odps1.execute_sql(sql).open_reader() as reader:
    for record in reader:
        try:            
            name=record.mbbm            
#遍历出元数据信息        
            for table in odps.list_tables():                
#判断自定义表数据存在于元数据里面
                if name == table.name:            
                #print(name)
                #print (table.name)
#判断表是否分区
                    if table.schema.partitions:                        
                        a=list=[]                                                           
#遍历元数据的 partitions信息
                        for partition in table.partitions:                                                        
                            fq=str(partition.name)
                            name=str(project.name+'.'+table.name)
                            sj=str(partition.creation_time)
                            dx=str(partition.size)
                            smzq=str(partition.lifecycle)
                            list.append('('+name)
                            list.append(fq) 
                            list.append(sj) 
                            list.append(smzq)
                            list.append(dx+')')                                                                                                                        
                      #  return list
                            continue
                        sql='''insert into t_jl_sjtz_gxqk PARTITION(xmkj='dwd__dev')(bm,fq,cjsj,smzq,dx) values %s '''%str(str(str(str(a).replace('[\'(','(\'') ).replace(')\']','\')')).replace('\'(','(\'')).replace(')\'','\')')
                        print (sql)
                        odps1.execute_sql(sql)
        except Exception as e:
		print(e)                
                    #print str(str(str(str(a).replace('[\'(','(\'') ).replace(')\']','\')')).replace('\'(','(\'')).replace(')\'','\')')
                    #sql1='''select array(%s) '''%(str(a).replace("]",""))
                    #print(sql1)
                    #s=a.list.repalce('[','')
                    #print  (a)

联系我们

如果您对我们的服务有兴趣,请及时和我们联系!

服务热线:18288888888
座机:18288888888
传真:
邮箱:888888@qq.com
地址:郑州市文化路红专路93号