SCOOP and DEAP

[Jan]

Hi guys,

I have tried to use SCOOP to speed up my GA/DE optimization. When I use SCOOP via cmd (python -m scoop main.py) in some way scoop does not find my main.py script:

[2019-11-03 23:32:20,559] launcher  INFO    SCOOP 0.7 1.1 on win32 using Python 3.6.8 |Anaconda, Inc.| (default, Dec 30 2018, 18:50:55) [MSC v.1915 64 bit (AMD64)], API: 1013

[2019-11-03 23:32:20,559] launcher  INFO    Deploying 4 worker(s) over 1 host(s).

[2019-11-03 23:32:20,560] launcher  INFO    Worker distribution:

[2019-11-03 23:32:20,560] launcher  INFO       127.0.0.1:       3 + origin

Traceback (most recent call last):

  File "C:\...\Anaconda3\lib\runpy.py", line 193, in _run_module_as_main

    "__main__", mod_spec)

  File "C:\...\Anaconda3\lib\runpy.py", line 85, in _run_code

    exec(code, run_globals)

  File "C:\...\Anaconda3\lib\site-packages\scoop-0.7.1.1-py2.7.egg\scoop\bootstrap\__main__.py", line 302, in <module>

  File "C:\...\Anaconda3\lib\site-packages\scoop-0.7.1.1-py2.7.egg\scoop\bootstrap\__main__.py", line 92, in main

  File "C:\...\Anaconda3\lib\site-packages\scoop-0.7.1.1-py2.7.egg\scoop\bootstrap\__main__.py", line 290, in run

  File "C:\...\Anaconda3\lib\site-packages\scoop-0.7.1.1-py2.7.egg\scoop\bootstrap\__main__.py", line 271, in futures_startup

  File "C:\...\Anaconda3\lib\site-packages\scoop-0.7.1.1-py2.7.egg\scoop\futures.py", line 64, in _startup

  File "C:\...\Anaconda3\lib\site-packages\scoop-0.7.1.1-py2.7.egg\scoop\_control.py", line 253, in runController

  File "C:\...\Anaconda3\lib\site-packages\scoop-0.7.1.1-py2.7.egg\scoop\_control.py", line 127, in runFuture

  File "C:\...\Anaconda3\lib\runpy.py", line 261, in run_path

    code, fname = _get_code_from_file(run_name, path_name)

  File "C:\...\Anaconda3\lib\runpy.py", line 231, in _get_code_from_file

    with open(fname, "rb") as f:

FileNotFoundError: [Errno 2] No such file or directory: 'main.py'

[2019-11-03 23:32:22,932] launcher  (127.0.0.1:58691) INFO    Root process is done.

[2019-11-03 23:32:22,933] launcher  (127.0.0.1:58691) INFO    Finished cleaning spawned subprocesses.

It seems not to be a problem of SCOOP as when I parallelize other scripts without DEAP it works properly.

My main script has the following structure:

import random as rm
from timeit import default_timer as timer
# DEAP: Distributed Evolutionary Algorithms in Python
from deap import base
from deap import creator
from deap import tools
# SCOOP: Scalable COncurrent Operations in Python
from scoop import futures

def main():
    """Genetic algorithm (structural optimization) with differential evolution (heat duty optimization)"""
    rm.seed(64)
    # GA: Create GA classes
    creator.create('FitnessMin_ga', base.Fitness, weights=(1.0,))
    creator.create('Individual_ga', list, fitness=creator.FitnessMin_ga)
    # DE: Create DE classes
    creator.create('FitnessMin_de', base.Fitness, weights=(1.0, 1.0))
    creator.create('Individual_de', list, fitness=creator.FitnessMin_de)
    # GA: Define individuals of exchanger address matrices
    toolbox = base.Toolbox()
    toolbox.register('individual_ga', ga.initialize_individual, creator.Individual_ga)
    toolbox.register('population_ga', tools.initRepeat, list, toolbox.individual_ga)
    # Fitness function
    toolbox.register('evaluate_ga', ga.fitness_function) # probably de.fitness_function or not needed!!!
    toolbox.register('select_ga', tools.selRoulette, fit_attr='fitness')
    # toolbox.register('select', tools.selTournament, tournsize=self.tourn_size)
    toolbox.register('mate_ga', ga.crossover)
    toolbox.register('mutate_ga', ga.mutation)
    # GA: Constraints
    toolbox.decorate('evaluate_ga', tools.DeltaPenality(ga.constraint_stream_least_one_heat_exchanger, case_study.ga_penalty_total_annual_cost_value, ga.stream_least_one_heat_exchanger_penalty))
    toolbox.decorate('evaluate_ga', tools.DeltaPenality(ga.constraint_no_utility_connections, case_study.ga_penalty_total_annual_cost_value, ga.utility_connections_penalty))
    toolbox.decorate('evaluate_ga', tools.DeltaPenality(ga.constraint_number_of_splits, case_study.ga_penalty_total_annual_cost_value, ga.split_number_penalty))
    # SCOOP: Parallel computing
    toolbox.register("map", futures.map)
    # GA: Generate population
    population = toolbox.population_ga(n=ga.population_size)
    hall_of_fame = tools.HallOfFame(maxsize=ga.hall_of_fame_size)

    # GA / DE: Evaluate entire population
    fitness = list(map(toolbox.evaluate_ga, population))
    for individual, fit in zip(population, fitness):
        individual.fitness.values = fit
    # GA: Extract all fitesses
    fits = [individual.fitness.values[0] for individual in population]

    number_generation_ga = 0
    start = timer()
    while max(fits) < 100 and number_generation_ga < ga.number_generations:
        """Genetic algorithm"""
        number_generation_ga += 1
        print('-- GA: Generation %i --' % number_generation_ga)
        # Select the next generation individuals
        offspring = toolbox.select_ga(population, len(population))
        # GA: Clone selected individuals
        offspring = list(toolbox.map(toolbox.clone, offspring))
        # GA: Apply crossover and mutation on offspring
        for child1, child2 in zip(offspring[::2], offspring[1::2]):
            if rm.random() < ga.probability_crossover:
                toolbox.mate_ga(child1, child2)
                del child1.fitness.values
                del child2.fitness.values

        for mutant in offspring:
            if rm.random() < ga.probability_mutation:
                toolbox.mutate_ga(mutant)
                del mutant.fitness.values
        # Evaluate the individuals with an invalid fitness
        invalid_individual = [individual for individual in offspring if not individual.fitness.valid]
        fitness = toolbox.map(toolbox.evaluate_ga, invalid_individual)
        for individual, fit in zip(invalid_individual, fitness):
            individual.fitness.values = fit

        population[:] = offspring

        # Gather all the fitness's in one list and print the stats
        fits = [individual.fitness.values[0] for individual in population]

        # Update hall of fame
        if number_generation_ga == 1:
            hall_of_fame.update(population)
        if number_generation_ga != 1:
            old = sum(hall_of_fame[0].fitness.getValues())
            hall_of_fame.update(population)
            new = sum(hall_of_fame[0].fitness.getValues())
            assert old <= new


    print('-- End of evolution --')
    end = timer()
    print('Computation time: %s s' % (end - start))
    print('Hall of fame list:')
    for i in range(len(hall_of_fame)):
        print('GA: Individual %s, is %s with TAC: %s, DE:' % (i + 1, hall_of_fame[i], 1 / hall_of_fame[i].fitness.values[0]))


if __name__ == '__main__':
    main()