using environment for build

api.py

class EnvironmentConfig(BaseModel):
    environment: str = "development"
    
    @classmethod
    def from_env(cls, explicit_env: Optional[str] = None):
        """
        Create an instance of the config, prioritizing explicitly provided environment 
        over environment variable, with a default fallback.
        """
        # Check for explicitly provided environment
        if explicit_env:
            return cls(environment=explicit_env)
        
        # Try to read from environment variable
        env_from_os = os.getenv('APP_ENVIRONMENT')
        if env_from_os:
            return cls(environment=env_from_os)
        
        # Return default if no environment specified
        return cls()
    
    @field_validator('environment')
    @classmethod
    def validate_environment(cls, v: str):
        """
        Validate the environment value
        """
        valid_envs = ['development', 'staging', 'production']
        if v.lower() not in valid_envs:
            raise ValueError(f'Invalid environment. Must be one of {valid_envs}')
        return v.lower()

# Example usage
class MyApplication:
    def __init__(self, config: Optional[EnvironmentConfig] = None):
        # If no config provided, create a default one
        self.config = config or EnvironmentConfig.from_env()
        
        # Now you can use the validated environment
        print(f"Running in {self.config.environment} environment")

For that kind of environment validation, I'd expect it to happen within the framework, for example here

What public API will CFS expose? For example, pipelines need a get_data_sources and process methods, example

class Pipeline(AbstractPipeline):
  def get_data_sources(self) -> List[AbstractDataSource]:
     return [
       CFSSink(
           name='cfs_sink', # name is an identifier all data sources have
          # What does the user need to supply? i.e. perhaps information about features, schema, etc.
       ),
       ...
     ]

  def process(self) -> TableResult:
    # How is it used, for example could be something like this, but depends on the schema of the table / http API
    return self.execute_sql("INSERT INTO cfs_sink SELECT ...")

p.s. w.r.t. environment, the contract exposed to users is that:

• they supply environment variables in Leicester via conduis portal env
• those variables will be made available via the AbstractPipeline.get_app_env method

platform environment information such as bootstrap servers, etc, are not made available. I'm open to changing that if needed, though.