RL Policy Training - Double Array Converting Action Space

[negodfre]

Does anyone know how to convert the action from a np.array() to the Alpyne Action with "Double_Array"?

I keep getting an error that I need a "Double Array", but the received type is an "Object".
(see _convert_to_action(self, action) function in code below)

I tried a list, generator, etc. but I seem to get an error each time regarding the type of the variable.

Code:

from logging import DEBUG
from turtle import pos

from alpyne.client import alpyne_client
from alpyne.client.alpyne_client import AlpyneClient
from alpyne.client.utils import histogram_outputs_to_fake_dataset, limit
from alpyne.data.constants import RunStatus
from alpyne.data.spaces import Number, Observation

import pandas as pd
from pathlib import Path

import numpy as np

import os
import datetime
from datetime import datetime

from gym import spaces
import gym
import sys
import os

try:  # handle sb2 or sb3
    # v- stops pycharm from complaining
    # noinspection PyUnresolvedReferences
    from stable_baselines3 import PPO, A2C
    from stable_baselines3.common.env_util import make_vec_env
    from stable_baselines3.common.env_checker import check_env
    from stable_baselines3.common.vec_env import DummyVecEnv, SubprocVecEnv
    from stable_baselines3.common.utils import set_random_seed
    from stable_baselines3.common.callbacks import CheckpointCallback, EveryNTimesteps
except ModuleNotFoundError:
    try:
        # v- stops pycharm from complaining
        # noinspection PyUnresolvedReferences
        from stable_baselines import PPO2 as PPO
    except ModuleNotFoundError:
        raise ModuleNotFoundError("Need stable baselines to run this example")
    
from alpyne.client.alpyne_client import AlpyneClient
from alpyne.client.model_run import ModelRun
from alpyne.data.spaces import Observation, Action

from alpyne.client.abstract import BaseAlpyneEnv

class MyGameEnv(BaseAlpyneEnv):
    def __init__(self, sim: ModelRun):
        super().__init__(sim)

    def _get_observation_space(self) -> spaces.Space:
        #return spaces.Box(low=-200, high=200 , shape=(1,), dtype=np.float32)
        return spaces.Box(low=np.array([-200, -200, -200, -200, -200, -200, -200, -200, -200, -200]),
                           high=np.array([200, 200, 200, 200, 200, 200, 200, 200, 200, 200]), shape=(10,), dtype=np.float32)

    def _convert_from_observation(self, observation: Observation):
        ## Convert alpyne observation to observation for gym
        #return np.array(observation.rlDueDates)
        return observation.rlObsCollection

    def _get_action_space(self) -> spaces.Space:
        ## creates a binary array of 0,1 of length 10
        return spaces.Box(low=np.array([-5,-5,-5,-5,-5,-5,-5,-5,-5,-5]), 
                          high=np.array([5,5,5,5,5,5,5,5,5,5]), shape=(10,), dtype=np.float32)
    
    def _convert_to_action(self, action: np.ndarray) -> Action:
        ## converts spaces.Multibinary into an action that Alpyne understands
        #return Action(rlOrderIndices=[1,0,0,0,0,0,0,0,0,0])
        #print(f"Type: {type(action)}")
        #print(f"Action: {action}")
        #return Action(rlOrderActionColl = (float(elem) for elem in action))
        return Action(rlOrderActionColl = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0])

    def _calc_reward(self, observation: Observation) -> float:
        return observation.rlTotalMade

    def _terminal_alternative(self, observation: Observation) -> bool:
        pass

model_path = str(Path(__file__)/"model.jar")
client = AlpyneClient(str(model_path), port=51150, blocking=False, verbose=True)

if __name__ == '__main__':
    # create new model run from basic configuration
    cfg = client.configuration_template
    
    sim = client.create_reinforcement_learning(cfg)

    #wrap in our custom gym environment
    env = MyGameEnv(sim)
    
    # pass it to stable-baselines for RL training
    model = PPO('MlpPolicy', env, verbose=2)

    print("About to learn...")
    model.learn(10)
    model.save("Test_obsColl_actColl_PPO.zip")

Error:

alpyne.data.model_error.ModelError: ModelError[path=http://127.0.0.1:51150/runs/1/rl, status=400, error=Bad Request, msg=('{"Platform '
 'state":{"message":"{\\"reason\\":null,\\"next_event_time\\":0.91042864820733,\\"episode_count\\":2,\\"step_count\\":1,\\"model_time\\":0.31338588671951706}","status":"PAUSED"},"Exception '
 'message":"Expected data element \'rlOrderActionColl\' to be declared as (or '
 'less than) type \'DOUBLE_ARRAY\'; received type \'OBJECT\'"}')]

[negodfre]

I was able to resolve this issue by adding a DOUBLE_ARRAY in constants.py
Within the Enum InputTypes, I added a DOUBLE_ARRAY type, added DOUBLE_ARRAY return for from_value() function, and DOUBLE_ARRAY check for validate() function.

Additionally, when using Action(), I used the optional dictionary instead of the kwargs to specify the name of the variable, the type (aka ~ "DOUBLE_ARRAY"), value, and I left the units blank.

Also, I would note that improperly defining the observation or action space sometimes resulted in an infinite loop with "wait_for_sim" function. The status would always return "FRESH". Additionally, when improperly setting up the observation or action space, I sometimes received an error "Sequence is off: 0".