GoogleCodeExporter
commented
8 years ago Ok, I did reproduced problem with small script. And problem is with the think
you said about in #47 =).
I'm set active_downloads/active_seeds/active_limit = -1 for all tests
1) Add/del 5000 torrents 5 times without (!) any trackers being set == 30mb
memory
2) Add/del 5000 torrents 5 times with local tracker (udp://localhost..),
commented out "stop_announcing()" in tracker::abort, 5 secs for tracker
completion, stop and receive timeouts == 100mb memory (+20mb each run)
3) Add/del 5000 torrents 5 times with local tracker, 50 secs for tracker
timeouts == 170mb memory. And here is the problem place -- after 50 seconds
memory is not freed. I did wait 1 hour -- no result, memory is still held.
4) Add/del 5000 torrents 5 times with local tracker, 1 sec for tracker timeouts
~= 60mb memory, it varies from run to run (40..80mb).
So, the faster tracker you have -- less memory you will use :).
And here is the script I use:
#!/usr/bin/env python
import libtorrent as lt
import sys
import tempfile
import shutil
import time
import os
import subprocess as sp
#TRACKER_LIST = [['udp://cmsearch01.<BIG_COMPANY>.ru:2399']]
TRACKER_LIST = [['udp://localhost:6969']]
#TRACKER_LIST = []
def memusage(s):
proc = sp.Popen(['ps', 'auxf', '-U', 'mocksoul'], stdout=sp.PIPE)
for line in proc.stdout.readlines()[1:]:
items = line.split()
items[10] = ' '.join(items[10:])
items = items[:11]
user, pid, pcpu, pmem, vsz, rss, tt, stat, started, time, command = items
if int(pid) == os.getpid():
print '%s: %s/%s mem, %s%% CPU' % (s, vsz, rss, pcpu)
def add_torrents(tmpdir, session, cnt):
torrents = {}
ps = []
for i in range(cnt):
#import time
#time.sleep(0.001)
testfile = 'file%d' % i
#os.system('dd if=/dev/urandom of=%s/%s bs=16k count=1 >/dev/null 2>&1' % (tmpdir, testfile))
fs = lt.file_storage()
lt.add_files(fs, '%s/%s' % (tmpdir, testfile), 0)
torrent = lt.create_torrent(fs)
lt.set_piece_hashes(torrent, tmpdir)
torrent_info_dict = torrent.generate()
torrent_info_dict['announce-list'] = TRACKER_LIST
torrent_info = lt.torrent_info(torrent_info_dict)
add_params = {
'save_path': tmpdir,
'storage_mode': lt.storage_mode_t.storage_mode_sparse,
'paused': False,
'auto_managed': True,
'ti': torrent_info,
}
ps.append(add_params)
for add_params in ps:
handle = session.add_torrent(add_params)
torrents[str(add_params['ti'].info_hash())] = handle
print 'yes', len(torrents)
return torrents
def wait_torrents(torrents):
while not all((t.is_seed() for t in torrents.values())):
print 'wait', len([t for t in torrents.values() if not t.is_seed()])
time.sleep(0.1)
def remove_torrents(torrents, session):
for hdl in torrents.values():
session.remove_torrent(hdl)
def read_alerts(session):
alert = session.pop_alert()
read = 0
while alert:
#print alert, alert.message()
alert = session.pop_alert()
read += 1
print 'read %d alerts' % read
def main():
memusage('start')
try:
#dir = tempfile.mkdtemp(prefix='tleaktmp')
dir = '/tmp/tleaktmp6OhFYo'
print 'Using temp directory: %s' % dir
session = lt.session()
session.set_alert_queue_size_limit(1024 * 1024)
sts = lt.session_settings()
sts.tracker_completion_timeout = 60
sts.tracker_receive_timeout = 60
sts.stop_tracker_timeout = 60
sts.active_downloads = -1
sts.active_seeds = -1
sts.active_limit = -1
sts.udp_tracker_token_expiry = 120
sts.min_announce_interval = 1
sts.inactivity_timeout = 60
sts.connection_speed = 10
sts.allow_multiple_connections_per_ip = True
sts.max_out_request_queue = 128
sts.request_queue_size = 3
sts.use_read_cache = False
session.set_settings(sts)
#session.set_alert_mask(lt.alert.category_t.tracker_notification | lt.alert.category_t.status_notification)
#session.set_alert_mask(lt.alert.category_t.status_notification)
session.listen_on(16881, 16881, 'localhost')
memusage('session started')
#for i in range(5000):
# testfile = 'file%d' % i
# os.system('dd if=/dev/urandom of=%s/%s bs=16k count=1 >/dev/null 2>&1' % (dir, testfile))
for i in range(5):
torrents = add_torrents(dir, session, 5000)
assert len(session.get_torrents()) == 5000
memusage('added %d torrents' % len(torrents))
read_alerts(session)
wait_torrents(torrents)
memusage('all seeding')
time.sleep(2)
read_alerts(session)
remove_torrents(torrents, session)
memusage('all removed')
assert len(session.get_torrents()) == 0
read_alerts(session)
while 1:
memusage('waiting...')
time.sleep(3)
finally:
print 'remove tree'
#shutil.rmtree(dir)
if __name__ == '__main__':
main()
Tune, "dir" in main() and make there 5000 files (file0 -> file5000) there
before run.
Here is how looking last few lines of output:
all seeding: 154344/113928 mem, 86.4% CPU
read 0 alerts
all removed: 178024/137984 mem, 83.1% CPU
read 0 alerts
yes 5000
added 5000 torrents: 181360/140652 mem, 84.9% CPU
read 0 alerts
wait 3050
wait 1687
wait 144
all seeding: 181096/140708 mem, 87.4% CPU
read 0 alerts
all removed: 209640/168532 mem, 82.4% CPU
read 0 alerts
waiting...: 209640/168532 mem, 82.5% CPU
waiting...: 209640/168532 mem, 76.6% CPU
waiting...: 209640/168532 mem, 71.4% CPU
waiting...: 209640/168532 mem, 66.8% CPU
(wait a lot)
waiting...: 209640/168532 mem, 02.9% CPU
<CTRL-C>, <CTRL-Z>, kill -kill % :)Original comment by mocksoul on 25 Nov 2010 at 5:17
Original issue reported on code.google.com by
mocksoulon 15 Nov 2010 at 2:33