Python : 计算大文件MD5值

By | 2012/09/04

 

buffer = 8192 来自网络,但不知道是怎么实践出来的。

 

from hashlib import md5
import time
import os

def calMD5(str):
    m = md5()
    m.update(str)

    return m.hexdigest()

def calMD5ForFile(file):
    statinfo = os.stat(file)

    if int(statinfo.st_size)/(1024*1024) >= 1000 :
        print "File size > 1000, move to big file..."
        return calMD5ForBigFile(file)

    m = md5()
    f = open(file, 'rb')
    m.update(f.read())
    f.close()

    return m.hexdigest()

def calMD5ForFolder(dir,MD5File):
    outfile = open(MD5File,'w')
    for root, subdirs, files in os.walk(dir):
        for file in files:
            filefullpath = os.path.join(root, file)
            """print filefullpath"""

            filerelpath = os.path.relpath(filefullpath, dir)
            md5 = calMD5ForFile(filefullpath)
            outfile.write(filerelpath+' '+md5+"\n")
    outfile.close()

def calMD5ForBigFile(file):
    m = md5()
    f = open(file, 'rb')
    buffer = 8192    # why is 8192 | 8192 is fast than 2048

    while 1:
        chunk = f.read(buffer)
        if not chunk : break
        m.update(chunk)

    f.close()
    return m.hexdigest()

if __name__ == "__main__":
    #print calMD5("Hello World!")

    t = time.time()
    print calMD5ForFile("E:\\OS\\ubuntu-11.04-desktop-i386.iso")
    print time.time() - t
    t = time.time()
    print calMD5ForBigFile("E:\\OS\\ubuntu-11.04-desktop-i386.iso")
    print time.time() - t,"\n"

    t = time.time()
    print calMD5ForFile("E:\\OS\\ubuntu-12.04-desktop-amd64.iso")
    print time.time() - t
    t = time.time()
    print calMD5ForBigFile("E:\\OS\\ubuntu-12.04-desktop-amd64.iso")
    print time.time() - t,"\n"

    t = time.time()
    print calMD5ForFile("D:\\Virtual Machines\\Ubuntu 64-bit\\Ubuntu 64-bit-s001.vmdk")
    print time.time() - t
    t = time.time()
    print calMD5ForBigFile("D:\\Virtual Machines\\Ubuntu 64-bit\\Ubuntu 64-bit-s001.vmdk")
    print time.time() - t,"\n"

 

 

PS: 来张图纪念下未优化前的内存使用率:

memory usage

 

 

Leave a Reply

Your email address will not be published. Required fields are marked *