def create_agents(self, generator):
        """
        Given information on a set of countries and a generator function,
        generate the agents and assign the results to ``self.agents``.
 
        :type generator: DataFrame,str,int
        :param generator: A function which generates the agents.
        """
        self.generator = generator
        country_array = pd.concat([pd.Series([c] * k["Population"]) for c, k in self.df.iterrows()])
        country_array.index = range(len(country_array))
        # Garbage collect before creating new processes.
        gc.collect()
        self.agents = pd.concat(
            self.pool.imap(self._gen_agents,
                           np.array_split(country_array, self.processes * self.splits))
        )
        self.agents.index = range(len(self.agents))

def create_agents(self, self.processes * self.splits))
        )
        self.agents.index = range(len(self.agents))

def test_latlon2pix_internals(pix_size_single, origin_point, is_flipped,
                              num_chunks, chunk_position):
 
    img = make_image(pix_size_single,
                     num_chunks, chunk_position)
    chunk_idx = img.chunk_idx
    res_x = img._full_res[0]
    res_y = img._full_res[1]
    pix_size = (img.pixsize_x, img.pixsize_y)
    origin = (img._start_lon, img._start_lat)
 
    # +0.5 for centre of pixels
    lons = (np.arange(res_x) + 0.5) * pix_size[0] + origin[0]
    all_lats = (np.arange(res_y) + 0.5) * pix_size[1] + origin[1]
    lats = np.array_split(all_lats, num_chunks)[chunk_idx]
 
    pix_x = np.arange(res_x)
    pix_y = np.arange(lats.shape[0])
 
    d = np.array([[a, b] for a in lons for b in lats])
    xy = img.lonlat2pix(d)
    true_xy = np.array([[a, b] for a in pix_x for b in pix_y])
    assert np.all(xy == true_xy)

def test_pix2latlong(pix_size_single, img._start_lat)
 
    true_lons = np.arange(res_x) * pix_size[0] + origin[0]
    all_lats = np.arange(res_y) * pix_size[1] + origin[1]
    true_lats = np.array_split(all_lats, num_chunks)[chunk_idx]
    true_d = np.array([[a, b] for a in true_lons for b in true_lats])
 
    pix_x = np.arange(res_x)
    pix_y = np.arange(img.resolution[1])  # chunk resolution
 
    xy = np.array([[a, b] for a in pix_x for b in pix_y])
 
    lonlats = img.pix2lonlat(xy)
    assert np.all(lonlats == true_d)

def transform(self, X):
        if self.tagger is None:
            raise ValueError("Must find_motifs before you can tag anything")
 
        logging.info("Tagging %s data with motifs using %d workers..." % (
            str(X.shape), self.n_jobs))
 
        if self.n_jobs > 1:
            pool = mp.ProcessingPool(self.n_jobs)
            splits = np.array_split(X, self.n_jobs)
            tag_lists = pool.map(self._tag_motifs, splits)
            tags = list(itertools.chain.from_iterable(tag_lists))
        else:
            tags = self._tag_motifs(X)
 
        logging.info("All motifs have been tagged")
        return self._sparsify_tags(tags)

def subset_iterator(X, m, repeats=1):
    ''''''
    Iterates over array X in chunks of m,repeat number of times.
    Each time the order of the repeat is randomly generated.
    ''''''
 
    N, dim = X.shape
    progress = tqdm(total=repeats * int(N / m))
 
    for i in range(repeats):
 
        indices = np.random.permutation(N)
 
        for idx in np.array_split(indices, N // m):
            yield X[idx][:]
            progress.update()
 
    progress.close()

def _split_into_groups(y, num_groups):
    groups = [[] for _ in range(num_groups)]
    group_index = 0
 
    for cls in set(y):
        this_cls_indices = np.where(y == cls)[0]
        num_cls_samples = len(this_cls_indices)
 
        num_cls_split_groups = ceil(num_cls_samples / 500)
        split = np.array_split(this_cls_indices, num_cls_split_groups)
 
        for cls_group in split:
            groups[group_index] = np.hstack((groups[group_index], cls_group))
            group_index = (group_index + 1) % num_groups
 
    return groups

def get_embedding_X(img):
    ''''''
            Args    : Numpy Images vector
            Returns : Embedded Matrix of length Samples,4096
    ''''''
    img = img.reshape((img.shape[0], img.shape[1], img.shape[2], 1))
    sess = tf.Session()
    imgs = tf.placeholder(tf.float32, [None, None, None])
    vgg = vgg16(imgs, ''/tmp/vgg16_weights.npz'', sess)
    embs = []
    cnt = 0
    for img_batch in np.array_split(img, img.shape[0] / 1000):
        emb = sess.run(vgg.emb, Feed_dict={vgg.imgs: img_batch})
        embs.extend(emb)
        cnt += 1
        progress = round(100 * (cnt * 1000 / img.shape[0]),2)
        if(progress%10 == 0):
          print progress
    embs = np.array(embs)
    print embs.shape
    embs = np.reshape(embs,(embs.shape[0],embs.shape[1] * embs.shape[2] * embs.shape[3]))
    return embs

def __init__(self, pobj, just_list = False, attr=''_grids'',
                 round_robin=False):
        ObjectIterator.__init__(self, just_list, attr=attr)
        # pobj has to be a ParallelAnalysisInterface,so it must have a .comm
        # object.
        self._offset = pobj.comm.rank
        self._skip = pobj.comm.size
        # Note that we''re doing this in advance,and with a simple means
        # of choosing them; more advanced methods will be explored later.
        if self._use_all:
            self.my_obj_ids = np.arange(len(self._objs))
        else:
            if not round_robin:
                self.my_obj_ids = np.array_split(
                                np.arange(len(self._objs)), self._skip)[self._offset]
            else:
                self.my_obj_ids = np.arange(len(self._objs))[self._offset::self._skip]

def iter_combinatorial_pairs(queue, num_examples, batch_size, interval,
                             num_classes, augment_positive=False):
    num_examples_per_class = num_examples // num_classes
    pairs = np.array(list(itertools.combinations(range(num_examples), 2)))
 
    if augment_positive:
        additional_positive_pairs = make_positive_pairs(
             num_classes, num_examples_per_class, num_classes - 1)
        pairs = np.concatenate((pairs, additional_positive_pairs))
 
    num_pairs = len(pairs)
    num_batches = num_pairs // batch_size
    perm = np.random.permutation(num_pairs)
    for i, batch_indexes in enumerate(np.array_split(perm, num_batches)):
        if i % interval == 0:
            x, c = queue.get()
            x = x.astype(np.float32) / 255.0
            c = c.ravel()
        indexes0, indexes1 = pairs[batch_indexes].T
        x0, x1, c0, c1 = x[indexes0], x[indexes1], c[indexes0], c[indexes1]
        t = np.int32(c0 == c1)  # 1 if x0 and x1 are same class,0 otherwise
        yield x0, t

def get_epoch_indexes(self):
        B = self.batch_size
        K = self.num_classes
        M = self.num_per_class
        N = K * M  # number of total examples
        num_batches = M * int(K // B)  # number of batches per epoch
 
        indexes = np.arange(N, dtype=np.int32).reshape(K, M)
        epoch_indexes = []
        for m in range(M):
            perm = np.random.permutation(K)
            c_batches = np.array_split(perm, num_batches // M)
            for c_batch in c_batches:
                b = len(c_batch)  # actual number of examples of this batch
                indexes_anchor = M * c_batch + m
 
                positive_candidates = np.delete(indexes[c_batch], axis=1)
                indexes_positive = positive_candidates[
                    range(b), np.random.choice(M - 1, size=b)]
 
                epoch_indexes.append((indexes_anchor, indexes_positive))
 
        return epoch_indexes

def pre_processing(self):
        """Provide same API as Model,we split data to K folds here.
        """
        if self.random:
            mask = np.random.permutation(self.train_x.shape[0])
            train_x = self.train_x[mask]
            train_y = self.train_y[mask]
        else:
            train_x = self.train_x[:]
            train_y = self.train_y[:]
 
        if self.select_train_method == ''step'':
            self.x_folds = [train_x[i::self.k_folds] for i in range(0, self.k_folds)]
            self.y_folds = [train_y[i::self.k_folds] for i in range(0, self.k_folds)]
        else:
            self.x_folds = np.array_split(train_x, self.k_folds)
            self.y_folds = np.array_split(train_y, self.k_folds)
 
 
        # for i in range(self.k_folds):
        #     self.x_folds[i] = self.train_x[0] + self.x_folds[i] + self.train_x[-1]
        #     self.y_folds[i] = self.train_y[0] + self.y_folds[i] + self.train_y[-1]

def Train(self, C, A, Y, SF):
        ''''''
        Train the classifier using the sample matrix A and target matrix Y
        ''''''
        C.fit(A, Y)
        YH = np.zeros(Y.shape, dtype = np.object)
        for i in np.array_split(np.arange(A.shape[0]), 32):   #Split up verification into chunks to prevent out of memory
            YH[i] = C.predict(A[i])
        s1 = SF(Y, YH)
        print(''All:{:8.6f}''.format(s1))
        ''''''
        ss = ShuffleSplit(random_state = 1151)  #Use fixed state for so training can be repeated later
        trn,tst = next(ss.split(A,Y))         #Make train/test split
        mi = [8] * 1                            #Maximum number of iterations at each iter
        YH = np.zeros((A.shape[0]),dtype = np.object)
        for mic in mi:                                      #Chunk size to split dataset for CV results
            #C.SetMaxIter(mic)                               #Set the maximum number of iterations to run
            #C.fit(A[trn],Y[trn])                           #Perform training iterations
        ''''''

def add_point(self, t, alt, az):
 
        self.window.append((t, az))
        if self._current_window_size() < self.window_duration:
            return
 
        points = np.array(self.window)
        steady, current = np.array_split(points, 2)
 
        _, steady_cube = self.create_cube(steady)
        timestamps, current_cube = self.create_cube(current)
 
        t = self.denoise_and_compare_cubes(steady_cube, current_cube)
        self.trigger_criterion.append(list(t))
        self.trigger_criterion_timestamps.append(list(timestamps))
 
        has_triggered = self.check_trigger(t)
        new_duration = self.window_duration - self.step
        self._reduce_to_duration(new_duration)

def predict(self):
        if os.path.exists(DATA_QUERIES_VECTOR_NPZ) and not FORCE_LOAD:
            print(''{}: loading precomputed data''.format(self.__class__.__name__))
            self.load_precomputed_data()
        else:
            self.precomputed_similarity()
 
        batch_size = 100
        batch_elements = math.ceil(self.queries_vector.shape[0] / batch_size)
        batch_queue = np.array_split(self.queries_vector.A, batch_elements)
        print("starting batch computation of Similarity and KNN calculation")
 
        # # multiple versions of calculating the prediction,some faster,some use more mem
 
        # prediction = self.multiprocessor_batch_calc(batch_queue)
        prediction = self.batch_calculation(batch_queue)
        # prediction = self.individual_calculation()
        # prediction = self.cosine_knn_calc()
        # prediction = self.custom_knn_calculation(prediction)
 
        train_avg_salary = sum(self.y_train) / len(self.y_train)
        cleaned_predictions = [x if str(x) != ''nan'' else train_avg_salary for x in prediction]
 
        return self.y_train, cleaned_predictions

def load_test_data(self):
        # Remove non-mat files,and perform ascending sort
        allfiles = os.listdir(self.data_dir)
        npzfiles = []
        for idx, f in enumerate(allfiles):
            if ".npz" in f:
                npzfiles.append(os.path.join(self.data_dir, f))
        npzfiles.sort()
 
        # Files for validation sets
        val_files = np.array_split(npzfiles, self.n_folds)
        val_files = val_files[self.fold_idx]
 
        print "\\n========== [Fold-{}] ==========\\n".format(self.fold_idx)
 
        print "Load validation set:"
        data_val, label_val = self._load_npz_list_files(val_files)
 
        return data_val, label_val

def __init__(self, X, kern, Xm):
 
        super(PITC, self).__init__("PITC")
        M = np.shape(Xm)[0]
        self.M = M
 
        start = time.time()
        X_split = np.array_split(X, M)
        self.kern = kern
        kern_blocks = np.zeros((M),dtype=object)
 
        for t in xrange(M):
            nyst = Nystrom(X_split[t], Xm, False)
            size = np.shape(X_split[t])[0]
            kern_blocks[t] = kern.K(X_split[t], X_split[t]) - nyst.precon  + (kern.noise)*np.identity(size)
 
        self.blocks = kern_blocks
        blocked = block_diag(*kern_blocks)
 
        self.nyst = Nystrom(X, False)
        self.precon = self.nyst.precon + blocked
        self.duration = time.time() - start

def __init__(self, False)
        self.precon = self.nyst.precon + blocked
        self.duration = time.time() - start

def _read_image_as_array(path, dtype, load_size, crop_size, flip):
    f = Image.open(path)
 
    A, B = numpy.array_split(numpy.asarray(f), 2, axis=1)
    if hasattr(f, ''close''):
        f.close()
 
    A = _resize(A, Image.BILINEAR, dtype)
    B = _resize(B, Image.NEAREST, dtype)
 
    sx, sy = numpy.random.randint(0, load_size-crop_size, 2)
    A = _crop(A, sx, sy, crop_size)
    B = _crop(B, crop_size)
 
    if flip and numpy.random.rand() > 0.5:
        A = numpy.fliplr(A)
        B = numpy.fliplr(B)
 
    return A.transpose(2, 0, 1), B.transpose(2, 1)

def setup_figure():
 
    f = plt.figure(figsize=(7, 5))
 
    mat_grid = plt.GridSpec(2, 6, .07, .52, .98, .95, .15, .20)
    mat_axes = [f.add_subplot(spec) for spec in mat_grid]
    sticks_axes, rest_axes = np.array_split(mat_axes, 2)
 
    scatter_grid = plt.GridSpec(1, .30, .49, .05)
    scatter_axes = [f.add_subplot(spec) for spec in scatter_grid]
 
    kde_grid = plt.GridSpec(1, .21, .05)
    kde_axes = [f.add_subplot(spec) for spec in kde_grid]
 
    cbar_ax = f.add_axes([.04, .62, .015, .26])
 
    return f, sticks_axes, rest_axes, scatter_axes, kde_axes, cbar_ax

def partitions(min_val, max_val, n):
 
    """
    Get start/stop boundaries for N partitions.
 
    Args:
        min_val (int): The starting value.
        max_val (int): The last value.
        n (int): The number of partitions.
    """
 
    pts = np.array_split(np.arange(min_val, max_val+1), n)
 
    bounds = []
    for pt in pts:
        bounds.append((int(pt[0]), int(pt[-1])))
 
    return bounds

def fit(self, y):
        """Fit a series of independent estimators to the dataset.
 
        Parameters
        ----------
        X : array,shape (n_samples,n_features,n_estimators)
            The training input samples. For each data slice,a clone estimator
            is fitted independently.
        y : array,)
            The target values.
 
        Returns
        -------
        self : object
            Return self.
        """
        self._check_Xy(X, y)
        self.estimators_ = list()
        # For fitting,the parallelization is across estimators.
        parallel, p_func, n_jobs = parallel_func(_sl_fit, self.n_jobs)
        estimators = parallel(
            p_func(self.base_estimator, split, y)
            for split in np.array_split(X, n_jobs, axis=-1))
        self.estimators_ = np.concatenate(estimators, 0)
        return self

def _transform(self, method):
        """Aux. function to make parallel predictions/transformation."""
        self._check_Xy(X)
        method = _check_method(self.base_estimator, method)
        if X.shape[-1] != len(self.estimators_):
            raise ValueError(''The number of estimators does not match ''
                             ''X.shape[2]'')
        # For predictions/transforms the parallelization is across the data and
        # not across the estimators to avoid memory load.
        parallel, n_jobs = parallel_func(_sl_transform, self.n_jobs)
        X_splits = np.array_split(X, axis=-1)
        est_splits = np.array_split(self.estimators_, n_jobs)
        y_pred = parallel(p_func(est, x, method)
                          for (est, x) in zip(est_splits, X_splits))
 
        if n_jobs > 1:
            y_pred = np.concatenate(y_pred, axis=1)
        else:
            y_pred = y_pred[0]
        return y_pred

def _yield_minibatches_idx(self, n_batches, data_ary, shuffle=True):
            indices = np.arange(data_ary.shape[0])
            if shuffle:
                indices = np.random.permutation(indices)
            if n_batches > 1:
                remainder = data_ary.shape[0] % n_batches
 
                if remainder:
                    minis = np.array_split(indices[:-remainder], n_batches)
                    minis[-1] = np.concatenate((minis[-1],
                                                indices[-remainder:]),
                                               axis=0)
                else:
                    minis = np.array_split(indices, n_batches)
 
            else:
                minis = (indices,)
 
            for idx_batch in minis:
                yield idx_batch

def test_mini_batch_k_means_random_init_partial_fit():
    km = MiniBatchKMeans(n_clusters=n_clusters, init="random", random_state=42)
 
    # use the partial_fit API for online learning
    for X_minibatch in np.array_split(X, 10):
        km.partial_fit(X_minibatch)
 
    # compute the labeling on the complete dataset
    labels = km.predict(X)
    assert_equal(v_measure_score(true_labels, labels), 1.0)