###########################################################################
###########################################################################
##
##  Apptainer Build File: Skyply
##  Function: Fully-integrated Simulation stack for WRF, CMAQ, & SCICHEM
##
##  Documentation: 
##
###########################################################################
###########################################################################    

Bootstrap: docker
From: rockylinux:9

%labels
    Author HPCStack
    Version 1.0
    Description "Rocky 9 base with development tools for WRF/CMAQ"

%environment
############################################################
## Compiler Settings
############################################################

# Intel Compiler
#export CC=icc
#export CXX=icpc
#export FC=ifort
#export F77=ifort

# GNU Compiler
export CC=gcc
export CXX=g++
export FC=gfortran
export F77=gfortran

############################################################
## General
############################################################
export HPC_PATH=/opt/hpc
export MODEL_PATH=/opt/models


############################################################
## OpenMPI Settings
############################################################
export MPI=/opt/hpc/mpi


############################################################
## OpenMP Settings
############################################################
export OMP_NUM_THREADS=1
export OMP_PROC_BIND=spread
export OMP_PLACES=cores 


############################################################
## HDF5 Settings
############################################################
export HDF5=$HPC_PATH/hdf5
export PHDF5=$HDF5


############################################################
## NetCDF Settings
############################################################
export NETCDF_INCDIR
export NETCDF_LIBDIR
export NETCDF=$HPC_PATH/netcdf
export NETCDF=$NETCDF


############################################################
## IO/API Settings
############################################################
export MPI_HOME=$HPC_PATH/mpi
export IOAPI=$HPC_PATH/ioapi


############################################################
## WRF Settings
############################################################
export WRF_DIR=$MODEL_PATH/wrf
export PATH=$MPI_HOME/bin:$NETCDF/bin:$PATH
export LD_LIBRARY_PATH=$MPI_HOME/lib:$HDF5/lib:$NETCDF/lib:$IOAPI/lib:$LD_LIBRARY_PATH

############################################################
## WPS Settings
############################################################
export WPS_DIR=$MODEL_PATH/wps

############################################################
## CMAQ Settings
############################################################
export CMAQ_DIR=$MODEL_PATH/cmaq


export JASPERLIB=/usr/lib64
export JASPERINC=/usr/include




%post
# Update system and enable CRB repository
dnf -y install epel-release
dnf config-manager --set-enabled crb
dnf -y update

# Install development packages
dnf -y install \
    gcc gcc-gfortran gcc-c++ \
    python3 python3-pip \
    R R-devel \
    git wget which file time \
    make cmake automake autoconf libtool \
    libcurl-devel \
    tar bzip2 bzip2-devel \
    perl tcsh csh m4 \
    jasper jasper-devel \
    libpng libpng-devel \
    zlib zlib-devel \
    libxml2 libxml2-devel \
    xz xz-devel \
    zstd \
    sudo \
    unzip \
    diffutils \
    patch \
    --allowerasing \
    && dnf clean all

mkdir -p /opt/src /opt/hpc /opt/models 

# -------------------------------------------------
# Create non-root user
# -------------------------------------------------

USERNAME="hpcuser"
USER_UID=1000
USER_GID=1000

# Create group and user
groupadd -g ${USER_GID} ${USERNAME}
useradd -m -u ${USER_UID} -g ${USER_GID} -s /bin/bash ${USERNAME}

# Set password (optional; often not needed for batch jobs)
echo "${USERNAME}:password" | chpasswd

# Give ownership of relevant directories
chown -R ${USERNAME}:${USERNAME} /opt/src


%post
# -------------------------------------------------
# Define package URLs
# -------------------------------------------------

OPENMPI_URL="https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.6.tar.gz"
HDF5_URL="https://support.hdfgroup.org/releases/hdf5/v1_14/v1_14_6/downloads/hdf5-1.14.6.tar.gz"
NETCDF_C_URL="https://downloads.unidata.ucar.edu/netcdf-c/4.10.0/netcdf-c-4.10.0.tar.gz"
NETCDF_FORTRAN_URL="https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.2.tar.gz"
IOAPI_URL="https://www.cmascenter.org/ioapi/download/ioapi-3.2.tar.gz"
WRF_URL="https://github.com/wrf-model/WRF/releases/download/v4.6.0/v4.6.0.tar.gz"
WPS_URL="https://github.com/wrf-model/WPS/archive/refs/tags/v4.6.0.tar.gz"

SRC_DIR="/opt/src"
mkdir -p ${SRC_DIR}

# -------------------------------------------------
# Package list (name + URL)
# -------------------------------------------------

packages=(
  "openmpi:${OPENMPI_URL}"
  "hdf5:${HDF5_URL}"
  "netcdf-c:${NETCDF_C_URL}"
  "netcdf-fortran:${NETCDF_FORTRAN_URL}"
  "ioapi:${IOAPI_URL}"
  "wrf:${WRF_URL}"
  "wps:${WPS_URL}"
)

# -------------------------------------------------
# Download loop
# -------------------------------------------------

for pkg in "${packages[@]}"; do
    NAME="${pkg%%:*}"
    URL="${pkg#*:}"

    echo "Downloading ${NAME}..."
    
    mkdir -p ${SRC_DIR}/${NAME}
    cd ${SRC_DIR}/${NAME}
    
    wget ${URL}
done

%post

    export HPC_PATH=/opt/hpc
    export MODEL_PATH=/opt/models

    export MPI_HOME=$HPC_PATH/mpi
    export HDF5=$HPC_PATH/hdf5
    export NETCDF=$HPC_PATH/netcdf
    export IOAPI=$HPC_PATH/ioapi

    export PATH=$MPI_HOME/bin:$NETCDF/bin:$PATH
    export LD_LIBRARY_PATH=$MPI_HOME/lib:$HDF5/lib:$NETCDF/lib:$LD_LIBRARY_PATH

# -------------------------------------------------
# Build HDF5 (parallel)
# -------------------------------------------------
    cd /opt/src
    tar -xzf hdf5-1.14.6.tar.gz
    cd hdf5-1.14.6

    CC=mpicc FC=mpif90 ./configure --prefix=$HDF5 --enable-parallel --enable-fortran 
    make -j$(nproc)
    make install

# -------------------------------------------------
# Build NetCDF-C
# -------------------------------------------------

    cd /opt/src
    export CC=/opt/hpc/mpi/bin/mpicc
    export CXX=/opt/hpc/mpi/bin/mpicxx
    export CPPFLAGS="-I/opt/hpc/hdf5/include"
    export LDFLAGS="-L/opt/hpc/hdf5/lib"
    export LD_LIBRARY_PATH=/opt/hpc/hdf5/lib:$LD_LIBRARY_PATH
    tar -xzf netcdf-c-4.10.0.tar.gz
    cd netcdf-c-4.10.0

    ./configure --prefix=/opt/hpc/netcdf --enable-parallel-tests
    make -j$(nproc)
    make install

# -------------------------------------------------
# Build NetCDF-Fortran
# -------------------------------------------------
    cd /opt/src
    tar -xzf v4.6.2.tar.gz
    cd netcdf-fortran-4.6.2

    CPPFLAGS="-I$NETCDF/include" \
    LDFLAGS="-L$NETCDF/lib" \
    ./configure --prefix=$NETCDF --enable-large-files

    make -j$(nproc)
    make install

# -------------------------------------------------
# Build OpenMPI
# -------------------------------------------------
cd /opt/src
    tar -xzf openmpi-4.1.6.tar.gz
    cd openmpi-4.1.6

    ./configure --prefix=$MPI_HOME
    make -j$(nproc)
    make install

# -------------------------------------------------
# Install WRF
# -------------------------------------------------

cd /opt/src
tar -xzf v4.6.0.tar.gz
mv WRFV4.6.0 /opt/models/wrf

cd /opt/models/wrf

printf "34\n1\n" | ./configure

ulimit -s unlimited

./compile em_real -j $(nproc) 2>&1 | tee compile.log || true

ls -lh main || true
ls -lh run || true

# -------------------------------------------------
# Install WPS
# -------------------------------------------------

cd /opt/src
wget https://github.com/wrf-model/WPS/archive/refs/tags/v4.6.0.tar.gz

# Extract WPS source
tar -xzf v4.6.0.tar.gz
mv WPS-4.6.0 /opt/models/wps

cd /opt/models/wps

# Tell WPS where WRF is installed
export WRF_DIR=/opt/models/wrf

# Configure automatically (choose Linux + gfortran)
printf "1\n" | ./configure

# Compile WPS
./compile 2>&1 | tee compile.log || true

# Verify binaries
ls -lh geogrid/src/geogrid.exe || true
ls -lh ungrib/src/ungrib.exe || true
ls -lh metgrid/src/metgrid.exe || true











# -------------------------
# Install I/O API
# -------------------------
git clone https://github.com/cjcoats/ioapi-3.2.git
cd ioapi-3.2

export FC=gfortran
export CC=gcc
export NETCDF=/usr

# Configure (Linux2_x86_64gfort is typical)
make config
# Choose: Linux2_x86_64gfort

make -j$(nproc)

echo "export IOAPI=/opt/ioapi-3.2" >> /environment    


# -------------------------
# Install CMAQ
# -------------------------
cd /opt
git clone https://github.com/USEPA/CMAQ.git
cd CMAQ

# Checkout stable release (optional)
# git checkout 5.3.3

echo "export CMAQ_HOME=/opt/CMAQ" >> /environment

# CMAQ build scripts expect tcsh
echo "CMAQ installed. Build manually via bldit scripts." > /opt/README