cfobel
commented
7 years ago Command
conda search -c microdrop-plugins --override-channels --jsonLog from line_profiler
Notes
The vast majority of time is spent on calls to linked(prefix) and extracted() (see conda/cli/main_search.py lines 161-162).
The following profiler output shows that these lines account for nearly 95% of the run time:
Total time: 17.7298 s
File: C:\Users\chris\Miniconda2\lib\site-packages\conda\cli\main_search.py
Function: execute_search at line 130
Line # Hits Time Per Hit % Time Line Contents
==============================================================
130 @profile
131 def execute_search(args, parser):
132 1 40 40.0 0.0 import re
133 1 61 61.0 0.0 from conda.resolve import Resolve
134
135 1 22 22.0 0.0 if args.reverse_dependency:
136 if not args.regex:
137 parser.error("--reverse-dependency requires at least one package name")
138 if args.spec:
139 parser.error("--reverse-dependency does not work with --spec")
140
141 1 17 17.0 0.0 pat = None
142 1 16 16.0 0.0 ms = None
143 1 17 17.0 0.0 if args.regex:
144 if args.spec:
145 ms = ' '.join(args.regex.split('='))
146 else:
147 regex = args.regex
148 if args.full_name:
149 regex = r'^%s$' % regex
150 try:
151 pat = re.compile(regex, re.I)
152 except re.error as e:
153 raise CommandArgumentError("Failed to compile regex pattern for "
154 "search: %(regex)s\n"
155 "regex error: %(regex_error)s",
156 regex=regex, regex_error=repr(e))
157
158 1 796 796.0 0.0 prefix = context.prefix_w_legacy_search
159
160 1 33 33.0 0.0 import conda.install
161
162 1 14164129 14164129.0 34.2 linked = conda.install.linked(prefix)
163 1 24749352 24749352.0 59.7 extracted = conda.install.extracted()
164
165 # XXX: Make this work with more than one platform
166 1 23 23.0 0.0 platform = args.platform or ''
167 1 17 17.0 0.0 if platform and platform != context.subdir:
168 args.unknown = False
169 1 29 29.0 0.0 ensure_use_local(args)
170 1 28 28.0 0.0 ensure_override_channels_requires_channel(args, dashc=False)
171 1 17 17.0 0.0 channel_urls = args.channel or ()
172 1 23 23.0 0.0 index = get_index(channel_urls=channel_urls, prepend=not args.override_channels,
173 1 18 18.0 0.0 platform=args.platform, use_local=args.use_local,
174 1 17 17.0 0.0 use_cache=args.use_index_cache, prefix=None,
175 1 1541795 1541795.0 3.7 unknown=args.unknown)
176
177 1 1543 1543.0 0.0 r = Resolve(index)
178
179 1 30 30.0 0.0 if args.canonical:
180 json = []
181 else:
182 1 21 21.0 0.0 json = {}
183
184 1 21 21.0 0.0 names = []
185 6 153 25.5 0.0 for name in sorted(r.groups):
186 5 117 23.4 0.0 if '@' in name:
187 continue
188 5 111 22.2 0.0 if args.reverse_dependency:
189 ms_name = ms
190 for pkg in r.groups[name]:
191 for dep in r.ms_depends(pkg):
192 if pat.search(dep.name):
193 names.append((name, Package(pkg, r.index[pkg])))
194 else:
195 5 103 20.6 0.0 if pat and pat.search(name) is None:
196 continue
197 5 104 20.8 0.0 if ms and name != ms.split()[0]:
198 continue
199
200 5 104 20.8 0.0 if ms:
201 ms_name = ms
202 else:
203 5 101 20.2 0.0 ms_name = name
204
205 5 5741 1148.2 0.0 pkgs = sorted(r.get_pkgs(ms_name))
206 5 136 27.2 0.0 names.append((name, pkgs))
207
208 1 23 23.0 0.0 if args.reverse_dependency:
209 new_names = []
210 old = None
211 for name, pkg in sorted(names, key=lambda x: (x[0], x[1].name, x[1])):
212 if name == old:
213 new_names[-1][1].append(pkg)
214 else:
215 new_names.append((name, [pkg]))
216 old = name
217 names = new_names
218
219 6 127 21.2 0.0 for name, pkgs in names:
220 5 105 21.0 0.0 if args.reverse_dependency:
221 disp_name = pkgs[0].name
222 else:
223 5 102 20.4 0.0 disp_name = name
224
225 5 104 20.8 0.0 if args.names_only and not args.outdated:
226 print(name)
227 continue
228
229 5 105 21.0 0.0 if not args.canonical:
230 5 112 22.4 0.0 json[name] = []
231
232 5 107 21.4 0.0 if args.outdated:
233 vers_inst = [dist[1] for dist in map(dist2quad, linked)
234 if dist[0] == name]
235 if not vers_inst:
236 continue
237 assert len(vers_inst) == 1, name
238 if not pkgs:
239 continue
240 latest = pkgs[-1]
241 if latest.version == vers_inst[0]:
242 continue
243 if args.names_only:
244 print(name)
245 continue
246
247 12 264 22.0 0.0 for pkg in pkgs:
248 7 176 25.1 0.0 dist = pkg.fn[:-8]
249 7 140 20.0 0.0 if args.canonical:
250 if not context.json:
251 print(dist)
252 else:
253 json.append(dist)
254 continue
255 7 141 20.1 0.0 if platform and platform != context.subdir:
256 inst = ' '
257 7 180 25.7 0.0 elif dist in linked:
258 5 103 20.6 0.0 inst = '*'
259 2 41 20.5 0.0 elif dist in extracted:
260 1 18 18.0 0.0 inst = '.'
261 else:
262 1 18 18.0 0.0 inst = ' '
263
264 7 5093 727.6 0.0 if not context.json:
265 print('%-25s %s %-15s %15s %-15s %s' % (
266 disp_name, inst,
267 pkg.version,
268 pkg.build,
269 Channel(pkg.channel).canonical_name,
270 disp_features(r.features(pkg.fn)),
271 ))
272 disp_name = ''
273 else:
274 7 161 23.0 0.0 data = {}
275 7 278 39.7 0.0 data.update(pkg.info)
276 7 157 22.4 0.0 data.update({
277 7 148 21.1 0.0 'fn': pkg.fn,
278 7 146 20.9 0.0 'installed': inst == '*',
279 7 150 21.4 0.0 'extracted': inst in '*.',
280 7 143 20.4 0.0 'version': pkg.version,
281 7 142 20.3 0.0 'build': pkg.build,
282 7 141 20.1 0.0 'build_number': pkg.build_number,
283 7 207425 29632.1 0.5 'channel': Channel(pkg.channel).canonical_name,
284 7 197 28.1 0.0 'full_channel': pkg.channel,
285 7 403 57.6 0.0 'features': list(r.features(pkg.fn)),
286 7 215 30.7 0.0 'license': pkg.info.get('license'),
287 7 213 30.4 0.0 'size': pkg.info.get('size'),
288 7 167 23.9 0.0 'depends': pkg.info.get('depends'),
289 7 384 54.9 0.0 'type': pkg.info.get('type')
290 })
291
292 7 171 24.4 0.0 if data['type'] == 'app':
293 data['icon'] = make_icon_url(pkg.info)
294 7 174 24.9 0.0 json[name].append(data)
295
296 1 725 725.0 0.0 if context.json:
297 1 776755 776755.0 1.9 stdout_json(json)
N.B., both of these functions employ memoization, BUT this does not help here. The results of the functions are only required once per call to conda search and so must be recomputed from scratch on each call.
In a Windows MicroDrop Conda environment,
condacommands seem to execute quite slowly.This issue proposes profiling various
condaoperations to determine performance bottlenecks.