Memory usage

[chron0]

Just let me thank you for developing and releasing this absolute gem of a tool first :) I was unhappy with the often failing distribution of global cloudmaps for years because I need a reliable source to feed https://github.com/chron0/xfce-planet.

After successfully testing and using cloudmap with gentoo on my local desktop for a couple of days I wanted to deploy it on our webserver to produce high resolution cloudmaps every 3 hours as an autonomous service, so that the xplanet community and whoever else can just download and use them as a free, alternative source without having the need to setup and run cloudmap themselves.

The box is basically a slice of sponsored kvm running Ubuntu 12.04.4 LTS (x86_64), but no matter what I try, I always get:

Traceback (most recent call last):
  File "create_map.py", line 347, in <module>
    main()
   File "create_map.py", line 278, in main
    SatelliteData.outwidth))
MemoryError

trace

stat("/etc/apt/apt.conf", 0x7fff3b40ece0) = -1 ENOENT (No such file or directory)
stat("/var/lib/dpkg/status", {st_mode=S_IFREG|0644, st_size=687788, ...}) = 0
stat("/usr/bin/dpkg", {st_mode=S_IFREG|0755, st_size=253424, ...}) = 0
stat("/etc/debian_version", {st_mode=S_IFREG|0644, st_size=11, ...}) = 0
stat("", 0x7fff3b40f030)                = -1 ENOENT (No such file or directory)
close(7)                                = 0
stat("/usr/lib/python2.7/dist-packages/apport/packaging", 0x7fff3b40f400) = -1 ENOENT (No such file or directory)
open("/usr/lib/python2.7/dist-packages/apport/packaging.so", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib/python2.7/dist-packages/apport/packagingmodule.so", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib/python2.7/dist-packages/apport/packaging.py", O_RDONLY) = 7
fstat(7, {st_mode=S_IFREG|0644, st_size=9179, ...}) = 0
open("/usr/lib/python2.7/dist-packages/apport/packaging.pyc", O_RDONLY) = 8
fstat(8, {st_mode=S_IFREG|0644, st_size=11186, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f452dbac000
read(8, "\3\363\r\n\2105\204Oc\0\0\0\0\0\0\0\0\3\0\0\0@\0\0\0s)\0\0\0d\0"..., 4096) = 4096
fstat(8, {st_mode=S_IFREG|0644, st_size=11186, ...}) = 0
read(8, "Return the package a file belong"..., 4096) = 4096
read(8, "kages/apport/packaging.pyt\22\0\0\0ge"..., 4096) = 2994
read(8, "", 4096)                       = 0
close(8)                                = 0
munmap(0x7f452dbac000, 4096)            = 0
close(7)                                = 0
close(6)                                = 0
close(5)                                = 0
close(4)                                = 0
stat("/usr/lib/python2.7/dist-packages/apport/gettext", 0x7fff3b419630) = -1 ENOENT (No such file or directory)
open("/usr/lib/python2.7/dist-packages/apport/gettext.so", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib/python2.7/dist-packages/apport/gettextmodule.so", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib/python2.7/dist-packages/apport/gettext.py", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib/python2.7/dist-packages/apport/gettext.pyc", O_RDONLY) = -1 ENOENT (No such file or directory)
close(3)                                = 0
getcwd("/root/tmp", 1026)               = 10
lstat("/root", {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
lstat("/root/tmp", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/root/tmp/create_map.py", {st_mode=S_IFREG|0755, st_size=13373, ...}) = 0
access("/root/tmp/create_map.py", X_OK) = 0
stat("/root/tmp/create_map.py", {st_mode=S_IFREG|0755, st_size=13373, ...}) = 0
write(2, "Traceback (most recent call last"..., 35Traceback (most recent call last):
) = 35
write(2, "  File \"create_map.py\", line 347"..., 46  File "create_map.py", line 347, in <module>
) = 46
open("create_map.py", O_RDONLY)         = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=13373, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f452dbac000
read(3, "#!/usr/bin/env python\r\n\r\nfrom PI"..., 4096) = 4096
read(3, "   look = True\r\n            i = "..., 4096) = 4096
read(3, "igParser.SafeConfigParser(\r\n    "..., 4096) = 4096
read(3, "    i = 1\r\n    for satellite in "..., 4096) = 1085
read(3, "", 4096)                       = 0
write(2, "    ", 4    )                     = 4
write(2, "main()", 6main())                   = 6
write(2, "\n", 1
)                       = 1
close(3)                                = 0
munmap(0x7f452dbac000, 4096)            = 0
write(2, "  File \"create_map.py\", line 278"..., 42  File "create_map.py", line 278, in main
) = 42
open("create_map.py", O_RDONLY)         = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=13373, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f452dbac000
read(3, "#!/usr/bin/env python\r\n\r\nfrom PI"..., 4096) = 4096
read(3, "   look = True\r\n            i = "..., 4096) = 4096
read(3, "igParser.SafeConfigParser(\r\n    "..., 4096) = 4096
write(2, "    ", 4    )                     = 4
write(2, "SatelliteData.outwidth))\n", 25SatelliteData.outwidth))
) = 25
close(3)                                = 0
munmap(0x7f452dbac000, 4096)            = 0
write(2, "MemoryError", 11MemoryError)             = 11
write(2, "\n", 1
)                       = 1
rt_sigaction(SIGINT, {SIG_DFL, [], SA_RESTORER, 0x7f452d780cb0}, {0x512110, [], SA_RESTORER, 0x7f452d780cb0}, 8) = 0
exit_group(1) 

meminfo

MemTotal:        2051316 kB
MemFree:          257892 kB
Buffers:          265460 kB
Cached:           629452 kB
SwapCached:            0 kB
Active:          1155832 kB
Inactive:         460668 kB
Active(anon):     776140 kB
Inactive(anon):    11708 kB
Active(file):     379692 kB
Inactive(file):   448960 kB
Unevictable:           0 kB
Mlocked:               0 kB
SwapTotal:             0 kB
SwapFree:              0 kB
Dirty:              1500 kB
Writeback:             0 kB
AnonPages:        721676 kB
Mapped:            60148 kB
Shmem:             66260 kB
Slab:             121908 kB
SReclaimable:     103400 kB
SUnreclaim:        18508 kB
KernelStack:        1736 kB
PageTables:        20520 kB
NFS_Unstable:          0 kB
Bounce:                0 kB
WritebackTmp:          0 kB
CommitLimit:     1025656 kB
Committed_AS:    1489560 kB
VmallocTotal:   34359738367 kB
VmallocUsed:       12312 kB
VmallocChunk:   34359723644 kB
HardwareCorrupted:     0 kB
AnonHugePages:         0 kB
HugePages_Total:       0
HugePages_Free:        0
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:       2048 kB
DirectMap4k:       46836 kB
DirectMap2M:     2050048 kB
DirectMap1G:           0 kB

I've tried upgrading all pip dependencies, removing ubuntu python packages that might collide, I must have rebuilt the whole pip tree 5 times by now but I see no light. Do you have any ideas what might cause this fail? Since cloudmap works without a glitch on my gentoo box I suppose it must be some dependency (and not a bug in cloudmap) but at this point I am running out of things I could try and I would really like to see this fly.

[jmozmoz]

The crash happens at this line:

    images = np.empty(shape=(5, 2, 
                             SatelliteData.outheight, 
                             SatelliteData.outwidth))

outheight and outwidth are the values height and width from the $HOME/.CreateCloudMap/CreateCloudMap.ini

So first question: Is there such a file for the user (root?) running the script? Second question: What are the values? Could you please try this in a python console:

import numpy as np
height = 1024
width = 2048
images = np.empty(shape=(5, 2, height, width)

(replace the values of height and width with the ones from your ini file.) The error message looks like there is really not enough memory (and not a problem of dependencies).

[chron0]

Damn, that is the only setting I didn't try because the resolution is kinda fixed to 5400x2700, to go along with the xfce-planet script and cloudmap failed even before connecting and downloading any images so I just assumed it couldn't really have been a memory problem :)

But, you nailed it. I added 2G more swap space, no more memory error, ran into some issues with PIL(low), had to

ln -s /usr/lib/x86_64-linux-gnu/libz.so /usr/lib
ln -s /usr/lib/x86_64-linux-gnu/libfreetype.so /usr/lib
ln -s /usr/lib/x86_64-linux-gnu/libjpeg.so /usr/lib

pip uninstall PIL
pip install PIL --upgrade

and voila : https://apollo.open-resource.org/clouds.jpg

Thanks again :)

[jmozmoz]

Commit c35f58f211fdcfffd5e95e4965ba6b48015d2e6c should reduce the required amount of memory by about a factor of 1/5.

[chron0]

Sweet, I'll check and report:

prior (from pip):

''' /usr/bin/time -v create_map -c CreateCloudMap.ini ('Download image date/time: ', '2014-06-27 06:00 UTC') Satellite file: /home/chrono/.xplanet/world/clouds/raw/2014_6_27_600_MTSAT2_4_S1.jpeg Satellite file: /home/chrono/.xplanet/world/clouds/raw/2014_6_27_600_MET7_2_S1.jpeg Satellite file: /home/chrono/.xplanet/world/clouds/raw/2014_6_27_600_MSG3_9_S1.jpeg Satellite file: /home/chrono/.xplanet/world/clouds/raw/2014_6_27_600_GOES13_4_S1.jpeg Satellite file: /home/chrono/.xplanet/world/clouds/raw/2014_6_27_600_GOES15_4_S1.jpeg finished Command being timed: "create_map -c CreateCloudMap.ini" User time (seconds): 108.82 System time (seconds): 5.03 Percent of CPU this job got: 88% Elapsed (wall clock) time (h:mm:ss or m:ss): 2:08.69 Average shared text size (kbytes): 0 Average unshared data size (kbytes): 0 Average stack size (kbytes): 0 Average total size (kbytes): 0 Maximum resident set size (kbytes): 9053840 Average resident set size (kbytes): 0 Major (requiring I/O) page faults: 81 Minor (reclaiming a frame) page faults: 1826452 Voluntary context switches: 2843 Involuntary context switches: 585 Swaps: 0 File system inputs: 33216 File system outputs: 12624 Socket messages sent: 0 Socket messages received: 0 Signals delivered: 0 Page size (bytes): 4096 Exit status: 0 '''

Running fresh git clone:

''' /usr/bin/time -v python create_map.py -c CreateCloudMap.ini Download image date/time: 2014-06-27 06:00 UTC Satellite file: ./2014_6_27_600_MTSAT2_4_S1.jpeg Satellite file: ./2014_6_27_600_MET7_2_S1.jpeg Satellite file: ./2014_6_27_600_MSG3_9_S1.jpeg Satellite file: ./2014_6_27_600_GOES13_4_S1.jpeg Satellite file: ./2014_6_27_600_GOES15_4_S1.jpeg finished Command being timed: "python create_map.py -c CreateCloudMap.ini" User time (seconds): 108.27 System time (seconds): 5.09 Percent of CPU this job got: 91% Elapsed (wall clock) time (h:mm:ss or m:ss): 2:03.54 Average shared text size (kbytes): 0 Average unshared data size (kbytes): 0 Average stack size (kbytes): 0 Average total size (kbytes): 0 Maximum resident set size (kbytes): 3582352 Average resident set size (kbytes): 0 Major (requiring I/O) page faults: 0 Minor (reclaiming a frame) page faults: 1821929 Voluntary context switches: 2477 Involuntary context switches: 274 Swaps: 0 File system inputs: 0 File system outputs: 12624 Socket messages sent: 0 Socket messages received: 0 Signals delivered: 0 Page size (bytes): 4096 Exit status: 0 '''

[maxsendq]

Hi, I have same issue on Windows and I am pushing to 8192x4096 and get Traceback (most recent call last): File "D:\temp\cloudimagesmap\create_map-script.py", line 9, in load_entry_point('CreateCloudMap==0.4.2', 'console_scripts', 'create_map')()

File "C:\Python27\lib\site-packages\cloudmap\create_map.py", line 278, in main

SatelliteData.outwidth))

ValueError: array is too big. Now I am not short of RAM or grunt, but I am on windows so getting stuff out of processes isn't great. I get the same in 0.5

Thanks

[jmozmoz]

Ok. I can reproduce the problem using windows.

[jmozmoz]

A work around to the problem seems to be to use http://www.cygwin.com/ But this might be just by chance (if you use the 32bit version).

If you create such large images, then a lot of memory is needed (about 2 GB which is just at the limit for 32bit processes). Now the better solution is to use 64bit Python. I was able to install it on Windows and using the 64bit version of numpy, scipy, matplotlib, pillow, and pip from here: http://www.lfd.uci.edu/~gohlke/pythonlibs/

Then I followed the instructions from http://blog.victorjabur.com/tag/python-express-2010-visual-build-compile-windows-executable/ to setup an environment so I could compile python modules using 64bit Microsoft Visual Studio (Express) 2008 Compiler. Then simple issuing

pip install CreateCloudMap

did the trick.

Here is the list of all installed python libraries, just in case I forgot something important above:

>pip freeze
CreateCloudMap==0.5.0
DateTime==4.0.1
MarkupSafe==0.23
Pillow==2.4.0
Pygments==1.6
configobj==5.0.5
configparser==3.3.0r2
ipython==2.1.0
matplotlib==1.3.1
numpy==1.8.1
pillowfight==0.2
pycairo==1.10.0
pyproj==1.9.3
pyreadline==2.0
pyresample==1.1.0
pytz==2014.4
pyzmq==14.3.1
requests==2.3.0
scipy==0.14.0
six==1.7.2
zope.interface==4.1.1

Of course, your Windows must be a 64bit version. (On stackoverflow you can find a link to download Visual Studio 2008 Express which is not linked directly on the MS website anymore. Newer version seem not to work.)

In my case the 32bit version is crashing within the pyresample library so I do not really know a way to solve this problem in a cleverer way (e.g. splitting up the images and do the resampling sequentially).

[maxsendq]

Joachim,

I started with the 64Bit Python and was have issues with getting the libraries installed, so actually moved to 32bit. I will follow what you suggest and I will come back to you.

Ta.

P please consider the environment before printing this e-mail. Did you know that IT are now fully ISO 3103 Compliant

From: Joachim [mailto:notifications@github.com] Sent: 27 June 2014 14:42 To: jmozmoz/cloudmap Cc: Michael Dear Subject: [SPAM] [HEADER CHECKING] - Re: [cloudmap] Memory usage (#2) - Email found in subject

A work around to the problem seems to be to use http://www.cygwin.com/ But this might be just by chance (if you use the 32bit version).

If you create such large images, then a lot of memory is needed (about 2 GB which is just at the limit for 32bit processes). Now the better solution is to use 64bit Python. I was able to install it on Windows and using the 64bit version of numpy, scipy, matplotlib, pillow, and pip from here: http://www.lfd.uci.edu/~gohlke/pythonlibs/http://www.lfd.uci.edu/%7Egohlke/pythonlibs/

Then I followed the instructions from http://blog.victorjabur.com/tag/python-express-2010-visual-build-compile-windows-executable/ to setup an environment so I could compile python modules using 64bit Microsoft Visual Studio (Express) 2008 Compiler. Then simple issuing

pip install CreateCloudMap

did the trick.

Here is the list of all installed python libraries, just in case I forgot something important above:

pip freeze

CreateCloudMap==0.5.0

DateTime==4.0.1

MarkupSafe==0.23

Pillow==2.4.0

Pygments==1.6

configobj==5.0.5

configparser==3.3.0r2

ipython==2.1.0

matplotlib==1.3.1

numpy==1.8.1

pillowfight==0.2

pycairo==1.10.0

pyproj==1.9.3

pyreadline==2.0

pyresample==1.1.0

pytz==2014.4

pyzmq==14.3.1

requests==2.3.0

scipy==0.14.0

six==1.7.2

zope.interface==4.1.1

Of course, your Windows must be a 64bit version. (On stackoverflow you can find a link to download Visual Studio 2008 Express which is not linked directly on the MS website anymore. Newer version seem not to work.)

In my case the 32bit version is crashing within the pyresample library so I do not really know a way to solve this problem in a cleverer way (e.g. splitting up the images and do the resampling sequentially).

—

[maxsendq]

Hi,

Yes switching to 64bit seems to work, I even pushed to 16384x8192, but it takes a long time to render. I have noticed that the timing if the script is great. eg I have asked it to download an update of the cloud maps, which I can see are @ Dundee, but the script keeps using 900 image (both 1200 and 1500 images are available). If I replace the images it will use them but it wont download them. I am guessing it is a timing issue. Below is output when run at 1634,

Create Cloud images 2048x1024 Download image date/time: 2014-07-02 09:00 UTC Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_MTSAT2_4_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_MET7_2_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_MSG3_9_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_GOES13_4_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_GOES15_4_S1.jpeg 4096x2048 Download image date/time: 2014-07-02 09:00 UTC Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_MTSAT2_4_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_MET7_2_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_MSG3_9_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_GOES13_4_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_GOES15_4_S1.jpeg 8192x4096 Download image date/time: 2014-07-02 09:00 UTC Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_MTSAT2_4_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_MET7_2_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_MSG3_9_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_GOES13_4_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_GOES15_4_S1.jpeg 16384x8192 Download image date/time: 2014-07-02 09:00 UTC Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_MTSAT2_4_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_MET7_2_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_MSG3_9_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_GOES13_4_S1.jpeg Satellite file: c:\temp\cloudimagesmap\2014_7_2_900_GOES15_4_S1.jpeg Press any key to continue . . .

[jmozmoz]

I just tested it and this is the output:

create_map
Download image date/time:  2014-07-02 15:00 UTC
Satellite file: c:\temp\xplanet\images\2014_7_2_1500_MTSAT2_4_S1.jpeg
Satellite file: c:\temp\xplanet\images\2014_7_2_1500_MET7_2_S1.jpeg
Satellite file: c:\temp\xplanet\images\2014_7_2_1500_MSG3_9_S1.jpeg
Satellite file: c:\temp\xplanet\images\2014_7_2_1500_GOES13_4_S1.jpeg
Satellite file: c:\temp\xplanet\images\2014_7_2_1500_GOES15_4_S1.jpeg
finished

So I close this issue, because the memory problem is explained (not necessarily solved). If there is really a timing issue, then please open a new issue.

[chron0]

Hmm, I had the same observation today, 1200 and 1500 were very late somehow, when you run the script, it just doesn't download anything.

[maxsendq]

I have opened a new issue