追踪代码申请的内存大小

tracemalloc

为python代码{%code%}添加如下代码：

1
2
3
4
5
6
7
8
9
10
import tracemalloc
tracemalloc.start()
f=open('mem_size.txt','w',encoding='utf8')
while-clause:
    {% code %}
    snapshot=tracemalloc.take_snapshot()
    top_stats=snapshot.statistics('lineno')
    for stat in top_stats:
    	f.write(str(stat)+'\n')
f.close()

追踪这片代码所涉及的内存申请过程，包括所申请内存的大小

对记录结果进行分析：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
from collections import defaultdict
import matplotlib.pyplot as plt

with open('test.txt','r',encoding='utf8') as f:
    contents=f.read()

contents=contents.strip().split('\n')
print(len(contents))
malloc_size=defaultdict(list)
for content in contents:
    units=content.split(' ')
    if units[-4]=='KiB,':
        malloc_size[' '.join(units[:-5])].append(float(units[-5][5:])*1024)
    elif units[-4]=='B,':
        malloc_size[' '.join(units[:-5])].append(float(units[-5][5:]))
    elif units[-4]=='MiB,':
        malloc_size[' '.join(units[:-5])].append(float(units[-5][5:])*1024*1024)
    else:
        print(units)
        continue

def isIncrement(ls,guanxing):
    if len(ls)<2:
        return False
    o=0
    score=0
    cha=[ls[i+1]-ls[i] for i in range(len(ls)-1)]
    for i in range(len(cha)):
        if o*guanxing+cha[i]>1:
            score+=1
        o=cha[i]
    if score/len(cha)>0.5:
        return True
    else:
        return False

plt.rcParams['figure.dpi'] = 300
plt.rcParams['font.sans-serif'] = ['SimHei']
for ms in malloc_size:
    # if ms.startswith('/home/tellw'):
    #     print(ms)
    #     plt.title(ms)
    #     plt.plot(list(range(len(malloc_size[ms]))),malloc_size[ms])
    #     plt.show()
    guanxing=0.1
    if isIncrement(malloc_size[ms],guanxing):
        print(ms)
        plt.title(ms)
        plt.plot(list(range(len(malloc_size[ms]))),malloc_size[ms])
        plt.show()

可以查看自己所写的代码所申请内存空间大小的变化情况

memory_profiler

1
2
3
4
5
6
7
8
9
import numpy as np
from memory_profiler import profile

@profile
def func():
	for i in range(3):
		print(np.random.rand(100000,10000))

func()

结果：

1
2
3
4
5
6
Line #    Mem usage    Increment  Occurrences   Line Contents
=============================================================
     4     52.8 MiB     52.8 MiB           1   @profile
     5                                         def func():
     6     53.0 MiB      0.0 MiB           4    for i in range(3):
     7     53.0 MiB      0.1 MiB           3            print(np.random.rand(10000,1000))

objgraph+gc

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# -*- coding: utf-8 -*-
import gc, time
class OBJ(object):
    pass

def show_cycle_reference():
    a, b = OBJ(), OBJ()
    a.attr_b = b
    b.attr_a = a

if __name__ == '__main__':
    gc.disable() # 这里是否disable事实上无所谓
    gc.set_debug(gc.DEBUG_COLLECTABLE|gc.DEBUG_SAVEALL)
    for _ in range(1):
        show_cycle_reference()
    gc.collect()
    for o in gc.garbage:
        print(o)
    import pdb;pdb.set_trace()
    time.sleep(5)

结果

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
gc: collectable <tuple 0x00000160C3B46020>
gc: collectable <cell 0x00000160C3D195D0>
gc: collectable <tuple 0x00000160C3D19D20>
gc: collectable <function 0x00000160C3D288B0>
gc: collectable <function 0x00000160C3D28820>
gc: collectable <cell 0x00000160C3D196F0>
gc: collectable <function 0x00000160C3D28700>
gc: collectable <cell 0x00000160C3D19840>
gc: collectable <cell 0x00000160C3D198D0>
gc: collectable <set 0x00000160C3D38200>
gc: collectable <tuple 0x00000160C3D48580>
gc: collectable <function 0x00000160C3D3EA70>
gc: collectable <function 0x00000160C3D28790>
gc: collectable <function 0x00000160C3D3EB00>
gc: collectable <tuple 0x00000160C3D19D80>
gc: collectable <dict 0x00000160C3D22F80>
gc: collectable <type 0x00000160C3C25FE0>
gc: collectable <staticmethod 0x00000160C3D19870>
gc: collectable <tuple 0x00000160C3D22940>
gc: collectable <member_descriptor 0x00000160C3D48500>
gc: collectable <member_descriptor 0x00000160C3D48AC0>
gc: collectable <member_descriptor 0x00000160C3D48B00>
gc: collectable <getset_descriptor 0x00000160C3D48B40>
gc: collectable <getset_descriptor 0x00000160C3D48B80>
gc: collectable <OBJ 0x00000160C3B6C8B0>
gc: collectable <OBJ 0x00000160C3B6E560>
gc: collectable <dict 0x00000160C3AE0700>
gc: collectable <dict 0x00000160C3AE0880>
('func', 'args', 'keywords', '__dict__', '__weakref__')
<cell at 0x00000160C3D195D0: type object at 0x00000160C3C25FE0>
(<cell at 0x00000160C3D195D0: type object at 0x00000160C3C25FE0>,)
<function partial.__new__ at 0x00000160C3D288B0>
<function partial.__call__ at 0x00000160C3D28820>
<cell at 0x00000160C3D196F0: str object at 0x00000160C3D217B0>
<function partial.__repr__ at 0x00000160C3D28700>
<cell at 0x00000160C3D19840: set object at 0x00000160C3D38200>
set()
(<cell at 0x00000160C3D196F0: str object at 0x00000160C3D217B0>, <cell at 0x00000160C3D19840: set object at 0x00000160C3D38200>, <cell at 0x00000160C3D198D0: function object at 0x00000160C3D28700>)
<function partial.__repr__ at 0x00000160C3D3EA70>
<function partial.__setstate__ at 0x00000160C3D3EB00>
n    ', '__slots__': ('func', 'args', 'keywords', '__dict__', '__weakref__'), '__new__': <staticmethod(<function partial.__new__ at 0x00000160C3D288B0>)>, '__call__': <function partial.__call__ at 0x00000160C3D28820>, '__repr__': <function partial.__repr__ at 0x00000160C3D3EA70>, '__reduce__': <function partial.__reduce__ at 0x00000160C3D28790>, '__setstate__': <function partial.__setstate__ at 0x00000160C3D3EB00>, 'args': <member 'args' of 'partial' objects>, 'func': <member 'func' of 'partial' objects>, 'keywords': <member 'keywords' of 'partial' objects>, '__dict__': <attribute '__dict__' of 'partial' objects>, '__weakref__': <attribute '__weakref__' of 'partial' objects>}
<class 'functools.partial'>
<staticmethod(<function partial.__new__ at 0x00000160C3D288B0>)>
(<class 'functools.partial'>, <class 'object'>)
<member 'args' of 'partial' objects>
<member 'func' of 'partial' objects>
<member 'keywords' of 'partial' objects>
<attribute '__dict__' of 'partial' objects>
<attribute '__weakref__' of 'partial' objects>
<__main__.OBJ object at 0x00000160C3B6C8B0>
<__main__.OBJ object at 0x00000160C3B6E560>
{'attr_b': <__main__.OBJ object at 0x00000160C3B6E560>}
{'attr_a': <__main__.OBJ object at 0x00000160C3B6C8B0>}
-> time.sleep(5)

在gdb输入处键入

1
2
(Pdb) import objgraph
(Pdb) objgraph.show_backrefs(objgraph.at(0x00000160C3B6C8B0),5,filename='obj.dot')

得到描述对象之间关系（引用链）的图像

objgraph.show_growth

通过两次objgraph.show_growth()的调用环境差异得到内存中增加的对象。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# -*- coding: utf-8 -*-
import objgraph

_cache = []

class OBJ(object):
    pass

def func_to_leak():
    o  = OBJ()
    _cache.append(o)
    # do something with o, then remove it from _cache

    if True: # this seem ugly, but it always exists
        return
    _cache.remove(o)

if __name__ == '__main__':
    objgraph.show_growth()
    try:
        func_to_leak()
    except:
        pass
    print('after call func_to_leak')
    objgraph.show_growth()

结果

1
2
3
4
5
6
7
8
9
10
11
12
function                       2884     +2884
dict                           1623     +1623
tuple                          1399     +1399
wrapper_descriptor             1168     +1168
ReferenceType                   864      +864
method_descriptor               799      +799
builtin_function_or_method      786      +786
getset_descriptor               474      +474
type                            448      +448
cell                            383      +383
after call func_to_leak
OBJ        1        +1

在调用完func_to_leak()之后，内存中还存在着OBJ实例

objgraph.show_most_common_types给出程序上下文中占用内存的数量较多的变量类型，默认给出最多的前十种

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# -*- coding: utf-8 -*-
import objgraph, gc
class OBJ(object):
    pass

def show_cycle_reference():
    a, b = OBJ(), OBJ()
    a.attr_b = b
    b.attr_a = a

if __name__ == '__main__':
    gc.disable()
    for _ in range(50):
        show_cycle_reference()
    objgraph.show_most_common_types(20)

输出结果

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
function                   2884
dict                       1795
tuple                      1616
wrapper_descriptor         1168
ReferenceType              864
method_descriptor          799
builtin_function_or_method 786
getset_descriptor          474
type                       448
cell                       382
member_descriptor          361
list                       243
module                     150
ModuleSpec                 149
classmethod                118
property                   104
SourceFileLoader           102
OBJ                        100
frozenset                  100
set                        95

注释掉gc.disable语句，输出结果（30种最多的变量类型）

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
function                   2884
dict                       1741
tuple                      1497
wrapper_descriptor         1168
ReferenceType              864
method_descriptor          799
builtin_function_or_method 786
getset_descriptor          474
type                       448
cell                       382
member_descriptor          361
list                       243
module                     150
ModuleSpec                 149
classmethod                118
property                   104
SourceFileLoader           102
frozenset                  100
set                        95
staticmethod               88
_tuplegetter               79
_abc_data                  75
_NamedIntConstant          74
ABCMeta                    67
OBJ                        46
_SpecialGenericAlias       38
classmethod_descriptor     30
method                     27
Pattern                    23
FileFinder                 17

可以看到，有一半的OBJ实例没有被回收，存在内存泄漏

pympler

1
2
3
4
5
6
7
8
9
10
11
12
from pympler import tracker,muppy,summary
import gc
tr=tracker.SummaryTracker()
print('memory total')
all_objects=muppy.get_objects()
sum1=summary.summarize(all_objects)
summary.print_(sum1)
while_clause:
    gc.collect()
    print('memory difference')
    tr.print_diff()
    loop_func()

打印结果

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
memory total
                       types |   # objects |   total size
============================ | =========== | ============
                         str |      460242 |     58.50 MB
                        code |       82833 |     32.64 MB
                        dict |       79885 |     30.18 MB
                        list |      254209 |     19.04 MB
                        type |       10445 |     12.90 MB
                       tuple |      105130 |      7.14 MB
                         set |        4085 |      5.96 MB
     collections.OrderedDict |        6755 |      3.29 MB
           inspect.Parameter |       29776 |      1.82 MB
       weakref.ReferenceType |       16794 |      1.28 MB
                 abc.ABCMeta |         825 |      1.24 MB
                        cell |       30656 |      1.17 MB
                         int |       41339 |      1.13 MB
  builtin_function_or_method |       14747 |      1.01 MB
         function (__init__) |        5452 |    809.28 KB
memory difference
                  types |   # objects |   total size
======================= | =========== | ============
                   list |       88009 |     18.97 MB
                    str |       92362 |      7.04 MB
                    int |       20073 |    549.00 KB
                   code |         479 |    159.77 KB
                   type |          61 |     87.15 KB
                   dict |         246 |     73.92 KB
                  tuple |         163 |      8.27 KB
  weakref.ReferenceType |          71 |      5.55 KB
    function (__init__) |          28 |      4.16 KB
               property |          39 |      3.05 KB
                    set |          12 |      3.03 KB
      getset_descriptor |          47 |      2.94 KB
    function (__repr__) |          19 |      2.82 KB
      collections.deque |           3 |      2.23 KB
              frozenset |           7 |      1.98 KB
2024-11-09 17:42:29.005537: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:465] Loaded cuDNN version 8907
memory difference
                                                           types |   # objects |   total size
================================================================ | =========== | ============
                                                   numpy.ndarray |           8 |      4.99 MB
                                                            list |       44109 |      3.74 MB
                                                             str |       44215 |      3.32 MB
                                                             int |       10073 |    275.54 KB
                                                             set |           8 |    194.69 KB
                                           weakref.ReferenceType |        2002 |    156.41 KB
                                      builtin_function_or_method |        1967 |    138.30 KB
                                                            dict |          41 |     10.74 KB
                                                           tuple |          80 |      4.25 KB
                     tensorflow.python.framework.ops.EagerTensor |          25 |      4.10 KB
                                                            type |           2 |      2.06 KB
  tensorflow.python.framework.cpp_shape_inference_pb2.HandleData |          24 |      1.88 KB
            tensorflow.python.framework.tensor_shape.TensorShape |          24 |      1.50 KB
                      keras.src.backend.tensorflow.core.Variable |          24 |      1.31 KB
    tensorflow.python.ops.resource_variable_ops.ResourceVariable |          24 |      1.31 KB
memory difference
          types |   # objects |   total size
=============== | =========== | ============
  numpy.ndarray |           0 |    493.87 KB
            str |           3 |    288     B
           list |           3 |    240     B
memory difference
          types |   # objects |   total size
=============== | =========== | ============
  numpy.ndarray |           0 |    278.12 KB
memory difference
          types |   # objects |      total size
=============== | =========== | ===============
  numpy.ndarray |           0 |   -857712     B

监测内存中变量类型的变化

guppy

安装pip install guppy3

1
2
3
4
5
6
from guppy import hpy
h=hpy()
while_clause:
    gc.collect()
    print(h.heap())
    loop_func()

打印结果

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
Partition of a set of 2022181 objects. Total size = 267902754 bytes.
 Index  Count   %     Size   % Cumulative  % Kind (class / dict of class)
     0 584430  29 71918335  27  71918335  27 str
     1 100645   5 38919944  15 110838279  41 types.CodeType
     2 338727  17 26151200  10 136989479  51 tuple
     3 196620  10 19018176   7 156007655  58 bytes
     4 209933  10 15817776   6 171825431  64 list
     5  37466   2 14702720   5 186528151  70 dict (no owner)
     6  95971   5 14587592   5 201115743  75 function
     7  10204   1 13534480   5 214650223  80 type
     8   4080   0  6264448   2 220914671  82 set
     9   4956   0  5125624   2 226040295  84 dict of module
<2535 more rows. Type e.g. '_.more' to view.>
2024-11-12 17:56:23.562654: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:465] Loaded cuDNN version 8907
Partition of a set of 2026043 objects. Total size = 273632823 bytes.
 Index  Count   %     Size   % Cumulative  % Kind (class / dict of class)
     0 584551  29 71927320  26  71927320  26 str
     1 100643   5 38919472  14 110846792  41 types.CodeType
     2 338659  17 26144944  10 136991736  50 tuple
     3 196619  10 19018000   7 156009736  57 bytes
     4 209943  10 15815584   6 171825320  63 list
     5  37469   2 14704376   5 186529696  68 dict (no owner)
     6  95823   5 14565096   5 201094792  73 function
     7  10205   1 13536168   5 214630960  78 type
     8   4088   0  6462272   2 221093232  81 set
     9     90   0  5249132   2 226342364  83 numpy.ndarray
<2538 more rows. Type e.g. '_.more' to view.>
Partition of a set of 2026043 objects. Total size = 274138191 bytes.
 Index  Count   %     Size   % Cumulative  % Kind (class / dict of class)
     0 584551  29 71927320  26  71927320  26 str
     1 100643   5 38919472  14 110846792  40 types.CodeType
     2 338659  17 26144944  10 136991736  50 tuple
     3 196619  10 19018000   7 156009736  57 bytes
     4 209943  10 15815584   6 171825320  63 list
     5  37469   2 14704376   5 186529696  68 dict (no owner)
     6  95823   5 14565096   5 201094792  73 function
     7  10205   1 13536168   5 214630960  78 type
     8   4088   0  6462272   2 221093232  81 set
     9     90   0  5754852   2 226848084  83 numpy.ndarray
<2538 more rows. Type e.g. '_.more' to view.>
Partition of a set of 2026043 objects. Total size = 274422759 bytes.
 Index  Count   %     Size   % Cumulative  % Kind (class / dict of class)
     0 584551  29 71927320  26  71927320  26 str
     1 100643   5 38919472  14 110846792  40 types.CodeType
     2 338659  17 26144944  10 136991736  50 tuple
     3 196619  10 19018000   7 156009736  57 bytes
     4 209943  10 15815584   6 171825320  63 list
     5  37469   2 14704376   5 186529696  68 dict (no owner)
     6  95823   5 14565096   5 201094792  73 function
     7  10205   1 13536168   5 214630960  78 type
     8   4088   0  6462272   2 221093232  81 set
     9     90   0  6039644   2 227132876  83 numpy.ndarray
<2538 more rows. Type e.g. '_.more' to view.>

结果比较粗糙，只能看到numpy.ndarray类型变量所占内存空间有增有减，其他类型变量所占内存空间没有变化，没什么参考意义，不好用

对打印结果进行统计，代码如下，提取关键打印内容和画图的代码跟下一节的结果分析代码相同

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
import json

with open('test1.txt','r',encoding='utf8') as f:
    contents=f.read()

lines=contents.strip().split('\n')
count=0
features={}
keys=[]
key_posts=['Count','Co%','Size','S%','Cumulative','Cu%']
for line in lines:
    line=line.strip()
    if line[0].isdigit():
        if line[0]=='0':
            count+=1
        parts=line.split()
        for i in range(6):
            key=f'{parts[7]}_{key_posts[i]}'
            if key in features:
                features[key][0].append(count)
                features[key][1].append(int(parts[i+1]))
            else:
                features[key]=[[count],[int(parts[i+1])]]
with open('test.json','w',encoding='utf8') as f:
    json.dump(features,f,ensure_ascii=False,indent=4)
print(f'共有{count}次统计')

mem_top

安装pip install mem_top

1
2
3
4
5
from mem_top import mem_top
while_clause:
    gc.collect()
    print(mem_top())
    loop_func()

打印结果

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

refs:
84842   <class 'set'> {'内当', '传圭', '过早', '司马', '牛郎织女', '远垂不', '大智如愚', '背本', '积雪', '亲操', '龃龉不', '结帐', '颠倒衣裳', '马革盛', '闲言淡',
45492   <class 'dict'> {'一丁不识': [['yī'], ['dīng'], ['bù'], ['shí']], '一丁点儿': [['yī'], ['dīng'], ['diǎn'], ['er']], '一不小心':
45492   <class 'dict'> {'一丁不识': [['yī'], ['dīng'], ['bù'], ['shí']], '一丁点儿': [['yī'], ['dīng'], ['diǎn'], ['er']], '一不小心':
22713   <class 'list'> [b'import', b'sys', 'sys', b'', 'sys', b'import', b'requests', 'requests', b'', 'requests', b'import
10000   <class 'list'> ['./A11_0', './A11_1', './A11_10', './A11_100', './A11_102', './A11_103', './A11_104', './A11_105', # 一万个文件名称
10000   <class 'list'> [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 2 # 一万个数据编号
7914    <class 'dict'> {9856192: <weakref at 0x704b9bab8d10; to 'type' at 0x9664c0 (type)>, 9829408: <weakref at 0x704b9bab
6730    <class 'dict'> {'get_logger': <function get_logger at 0x704b9a7deca0>, 'compat.v1.get_logger': <function get_logger
6702    <class 'list'> ['# Copyright 2015 The TensorFlow Authors. All Rights Reserved.\n', '#\n', '# Licensed under the Apa
6697    <class 'list'> ['# Copyright 2015 The TensorFlow Authors. All Rights Reserved.\n', '#\n', '# Licensed under the Apa

bytes:
4194520  {'内当', '传圭', '过早', '司马', '牛郎织女', '远垂不', '大智如愚', '背本', '积雪', '亲操', '龃龉不', '结帐', '颠倒衣裳', '马革盛', '闲言淡',
1922480  {'一丁不识': [['yī'], ['dīng'], ['bù'], ['shí']], '一丁点儿': [['yī'], ['dīng'], ['diǎn'], ['er']], '一不小心':
1922480  {'一丁不识': [['yī'], ['dīng'], ['bù'], ['shí']], '一丁点儿': [['yī'], ['dīng'], ['diǎn'], ['er']], '一不小心':
207616   {'sys': <module 'sys' (built-in)>, 'builtins': <module 'builtins' (built-in)>, '_frozen_importlib':
207616   {'get_logger': <function get_logger at 0x704b9a7deca0>, 'compat.v1.get_logger': <function get_logger
194680   [b'import', b'sys', 'sys', b'', 'sys', b'import', b'requests', 'requests', b'', 'requests', b'import
147544   {9856192: <weakref at 0x704b9bab8d10; to 'type' at 0x9664c0 (type)>, 9829408: <weakref at 0x704b9bab
131288   {'aten/atanh_/Meta', 'prims/igammac/CompositeExplicitAutograd', 'aten/_foreach_add.List/Meta', 'prim
131288   {'aten/atanh_/Meta', 'aten/_foreach_add.List/Meta', 'aten/bitwise_right_shift.Tensor/Meta', 'aten/lc
80056    ['./A11_0', './A11_1', './A11_10', './A11_100', './A11_102', './A11_103', './A11_104', './A11_105',

types:
210288   <class 'list'>
94866    <class 'function'>
70083    <class 'tuple'>
44454    <class 'dict'>
30584    <class 'cell'>
29776    <class 'inspect.Parameter'>
18627    <class 'weakref.ReferenceType'>
16396    <class 'builtin_function_or_method'>
12452    <class 'getset_descriptor'>
8791     <class 'type'>

mem_top里参数limit控制显示最高占用量的对象，refs是指包含最多引用的对象，bytes是指占用内存空间最多的对象，types是指数量最多的对象类型。代码实现细节参见denis-ryzhkov/mem_top。间隔一段时间长时间观测，就能发现端倪。python进程内存占用持续增高排查经验分享，mem_top统计整个进程的内存占用情况，只要程序A的代码不变且mem_top被包含在程序A的代码里，无论mem_top插在哪里都是相同的结果。

附上mem_top结果的分析代码，以观察什么因素在增高导致内存泄露

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
# 1. 提取要分析的关键内容
# with open('test.txt','r',encoding='utf8') as f:
#     contents=f.read()

# lines=contents.strip().split('\n')
# new_lines=[]
# for line in lines:
#     line=line.strip()
#     if len(line)==0:
#         continue
#     if not line.startswith('2024-'):
#         new_lines.append(line)

# with open('test1.txt','w',encoding='utf8') as f:
#     f.write('\n'.join(new_lines))






# 2. 得到统计结果并持久化
# import json

# with open('test1.txt','r',encoding='utf8') as f:
#     contents=f.read()

# lines=contents.strip().split('\n')
# count=0
# flag=1
# features={}
# keys=[] # keys变量好像没用
# for line in lines:
#     line=line.strip()
#     if line=='refs:':
#         flag=1
#         count+=1
#         keys=[]
#         continue
#     if line =='bytes:':
#         flag=2
#         keys=[]
#         continue
#     if line=='types:':
#         flag=3
#         keys=[]
#         continue
#     if flag==1:
#         parts=line.split()
#         num=int(parts[0])
#         key=''.join(parts[1:])+'refs'
#         if key in keys:
#             continue
#         else:
#             keys.append(key)
#         if key in features:
#             features[key][0].append(count)
#             features[key][1].append(num)
#         else:
#             features[key]=[[count],[num]]
#     elif flag==2:
#         parts=line.split()
#         num=int(parts[0])
#         key=''.join(parts[1:])+'bytes'
#         if key in keys:
#             continue
#         else:
#             keys.append(key)
#         if key in features:
#             features[key][0].append(count)
#             features[key][1].append(num)
#         else:
#             features[key]=[[count],[num]]
#     elif flag==3:
#         parts=line.split()
#         num=int(parts[0])
#         key=''.join(parts[1:])+'types'
#         if key in keys:
#             continue
#         else:
#             keys.append(key)
#         if key in features:
#             features[key][0].append(count)
#             features[key][1].append(num)
#         else:
#             features[key]=[[count],[num]]
# with open('test.json','w',encoding='utf8') as f:
#     json.dump(features,f,ensure_ascii=False,indent=4)
# print(f'共有{count}次统计')






# 3. 画图
import json
import matplotlib.pyplot as plt

features=None
with open('test.json','r',encoding='utf8') as f:
    features=json.load(f)

plt.rcParams['figure.dpi'] = 300
plt.rcParams['font.sans-serif'] = ['SimHei']
for key in features:
    plt.title(key)
    plt.plot(features[key][0],features[key][1])
    plt.show()

memprof

1
2
3
4
5
6
7
from memprof import *

@memprof(plot=True)
def func():
    do_sth()

func()

代码运行目录下会生成func.log和func.png（描述func中的占用内存空间超过1MB的变量的内存使用量随着时间的变化情况）

参考链接：

python3使用迭代生成器yield减少内存占用

避免python内存泄漏的有效方法

使用gc、objgraph干掉python内存泄露与循环引用！

记一次python内存泄露的解决过程

python 内存诊断

mem-top 0.2.1

Memprof

创建于202411052157，修改于202412012056