HORD-ISP

[ravi-kislaya]

How do you set the initial starting point for HORD-ISP? I was not able to find any help for this matter in pySOT too. Can you point out the lines of code which sets the initial starting point? Sorry, if my issue is already answered in the existing code. I was not able to locate it. Thanks in advance.

[ili3p]

The values of the initial starting points for each experiment are in the supplementary, download it from here: https://ilija139.github.io/pub/aaai-17-sup.pdf

The initial starting points are set via the extra parameter https://github.com/ilija139/HORD/blob/8eb8419ad28f4fd86e1664b9fe5e5d9a1fe80ef9/pySOT/src/sot_sync_strategies.py#L61

For example, like this:

import sys
sys.path.append('../pySOT/')
from src import *
from poap.controller import ThreadController, BasicWorkerThread
import numpy as np
import os.path
from pySOT_torch import TorchOptim
from datetime import datetime
import time

def main():
    log_file = os.path.splitext(__file__)[0]+"_"+sys.argv[3]+"_.log"
    millis = int(round(time.time() * 1000))
    print('Started: ' + str(datetime.now()) + ' (' + str(millis) + ')')
    if not os.path.exists("./logfiles"):
        os.makedirs("logfiles")
    if os.path.exists("./logfiles/"+log_file):
        os.remove("./logfiles/"+log_file)
    logging.basicConfig(filename="./logfiles/"+log_file,
                        level=logging.INFO)

    nthreads = int(sys.argv[1])
    maxeval = int(sys.argv[2])
    seed = sys.argv[3]
    server = sys.argv[4]

    np.random.seed(int(seed))

    print("\nNumber of threads: "+str(nthreads))
    print("Maximum number of evaluations: "+str(maxeval))
    print("Search strategy: Candidate DyCORS")
    print("Experimental design: Latin Hypercube")
    print("Surrogate: Cubic RBF")
    print('best\tf_eval_time\tresult\ttestset_result\tf_eval_count\twallclock_time\thyper-parameters')
    nsamples = nthreads

    data = TorchOptim(seed=seed, server=server)

    extra = np.atleast_2d([
        0.1,       # 'learningRate' 
        0.9,       # 'momentum' 
        0.0005,    # 'weightDecay'
        0.0001,    # 'learningRateDecay' 
        0.01,      # 'leakyReLU_fc1' 
        0.01,      # 'leakyReLU_fc2' 
        0.01,      # 'std_fc1' 
        0.01,      # 'std_fc2' 
        0.01,      # 'std_conv1' 
        0.01,      # 'std_conv2' 
        0.5,       # 'drop_rate1' 
        0.5,       # 'drop_rate2' 
        0.5,       # 'drop_rate3' 
        0.5,       # 'drop_rate4' 
        128,       # 'batchSize' 
        200,       # 'hiddenNodes_fc1' 
        256,       # 'hiddenNodes_fc2' 
        32,        # 'hiddenNodes_conv1' 
        64])       # 'hiddenNodes_conv2' 

    weights = np.array([1])

    # Create a strategy and a controller
    controller = ThreadController()
    controller.strategy = \
        SyncStrategyNoConstraints(
            worker_id=0, data=data,
            maxeval=maxeval, nsamples=nsamples,
            exp_design=LatinHypercube(dim=data.dim, npts=2*(data.dim+1)),
            response_surface=RBFInterpolant(surftype=CubicRBFSurface, maxp=maxeval),
            sampling_method=CandidateDYCORS(data=data, numcand=100*data.dim, weights=weights), 
            extra=extra)

    # Launch the threads and give them access to the objective function
    for _ in range(nthreads):
        worker = BasicWorkerThread(controller, data.objfunction)
        controller.launch_worker(worker)

    # Run the optimization strategy
    result = controller.run()

    print('Best value found: {0}'.format(result.value))
    print('Best solution found: {0}\n'.format(
        np.array_str(result.params[0], max_line_width=np.inf,
                     precision=5, suppress_small=True)))

    millis = int(round(time.time() * 1000))
    print('Ended: ' + str(datetime.now()) + ' (' + str(millis) + ')')

if __name__ == '__main__':
    main()