Script for automated delta backup to Amazon AWS S3 for Windows computers

BackupWin.ps1

<#
This is a simple script does delta backup of specified folders and uploads
them to Amazon S3 bucket.

Prerequisites
=============
1) installed 7zip archiver (https://www.7-zip.org/)

Installation
============

This script can be installed/run from any folder.
Just copy the script and run it (or schedule it using Windows Task Scheduler)

Usage
=====

1) starting with no parameters
   > .\BackupWin.ps1

   Script will try opening configuration file BackupWin.ini located in 
   current folder.

2) specifying configuration file
   > .\BackupWin.ps1 -c config_file_name

   Script will process all sections defined in provided configuration file.

3) specifying section
   > .\BackupWin.ps1 -s section_name

   Script will process specified section specified in configuration file.

4) specifying log file
   > .\BackupWin.ps1 -log log_file_name
  


Configuration file structure
============================

File contains one or more backup configurations. Each configuration
specifies input & output folder and names of include & exclude files.

Parameters:
a) archiver_type
   Optional. Archiver used for creating a backup. Default is 7-zip.
b) archiver_executable
   Optional. Full path to archiver executable. Default is C:\Program Files\7-zip\7z.exe
c) input_folder 
   Required. Folder from which files will be read into archive.
d) output_folder
   Required. Folder where archive will be created.
e) include
   Optional. File containing definitions of files to be included in archive. 
   Default is all files. See archiver documentation for details.
f) exclude
   Optional. File containing definitions of files which will be excluded from
   processing while creating an archive. Default is none.
   See archiver documentation for details.
g) destination_folder
   Required. Destination where created archive will be moved. 
   Has to be understandable for uploader.
h) destination_type
   Required.
i) uplader_params
   Optional. Additional commandline parameters for uploader responsible
   for moving aerchive to destination folder.
j) incexc_folder
   Optional. If specified, include & exclude files does not have to be
   specified as full name.
k) explicit_only
   Optional.
   Values: yes/no
   If yes, then section will not be processed when processing all sections without
   explicitly specifying section name.

Example Configuration File
==========================

[documents]
archiver_type=7-zip
archiver_executable=C:\Program Files\7-zip\7z.exe
input_folder=C:\Users\MyUser\Documents
output_folder=C:\Users\MyUser\BackupWin
include=C:\Users\MyUser\Documents\BackupWin\home-inc.txt
exclude=C:\Users\MyUser\Documents\BackupWin\home-exc.txt

Another Example (specifying defaults)
========================

[defaults]
archiver_type=7-zip
archiver_executable=C:\Program Files\7-zip\7z.exe
destination_folder=s3://my_bucket
destination_type=aws_s3
output_folder=C:\Users\MyUser\BackupWinCache
uploader_params=--profile my-backup

[documents]
input_folder=C:\Users\MyUser\Documents

[project1]
input_folder=C:\Users\MyUser\MyProjects\Project1

#>

param (
    # if parameter "-c" cfg file was not specified, folder BackupWin from user application data folder will be used
    [Parameter(Mandatory=$False)] [string]$c = ".\BackupWin.ini",
    # if parameter "-s" section was not specified, all sections from cfg file will be processed
	[Parameter(Mandatory=$False)] [string]$s,
    # if parameter "-log" was not specified, log file will be created in the current folder
	[Parameter(Mandatory=$False)] [string]$log = ".\BackupWin.log"

)

$defaultArchiver="C:\Program Files\7-Zip\7z.exe"

# if log file does not exist, then create one
if (!(Test-Path -Path $log)) {
    Write-Host "Logfile created $log"
    Add-Content -Path $log -Value ""
}

# get log file full name
$log = (Get-Item -Path $log).FullName

# number of started background jobs
$bgCount = 0
$bgUploaders = @()

function Log($level, $module, $message, $logFile) {
    if ([string]::IsNullOrEmpty($logFile)) { $logFile = $log }
    $dateString = Get-Date -Format G
    $line= $dateString+" "+$level.PadRight(5)+" ["+$module+"] "+$message
    Add-Content -path $logFile -value $line
    Write-Host $line
}

# fucntiopn for reading content of configuration ini file
function Get-IniContent ($filePath) {
    $ini = @{}
    switch -regex -file $filePath
    {
        "^\[(.+)\]" # Section
        {
            $section = $matches[1]
            $ini[$section] = @{}
            $CommentCount = 0
        }
        "^(;.*)$" # Comment
        {
            $value = $matches[1]
            $CommentCount = $CommentCount + 1
            $name = "Comment" + $CommentCount
            $ini[$section][$name] = $value
        } 
        "(.+?)\s*=(.*)" # Key
        {
            $name,$value = $matches[1..2]
            $ini[$section][$name] = $value
        }
    }
    return $ini
}

function Get-Archiver-Params-7Zip($inFld, $outFld, $incFile, $excFile, $archName, $deltaArchName) {
    # archive type - 7zip supports zip archives too, 
    # but they do not support "delete" commend when extracting delta archive,
    # so it is better to use 7z archives
    $arctype = '-t7z'

    # -s=off - switch off solid archives
    $addSwitches = "-ms=off"
    $fExt = '.7z'

    # update switches
    $upSwitches = 'p0q3x0z0w2y2'

    #if ([string]::IsNullOrEmpty($incFile)) { $incFile = "-i!*" }
    #if ([string]::IsNullOrEmpty($excFile)) { $excFile = "-x!huhu.huhu" } # dummy exclude file

    if (![string]::IsNullOrEmpty($incFile)) { $incParam = $('-ir@' + '"' + $incFile + '"') }
    if (![string]::IsNullOrEmpty($excFile)) { $excParam = $('-xr@' + '"' + $excFile + '"') }

    #return array containing parameters which will be passed to archiver
    return @("u", $arctype, $('"'+$archName+$fExt+'"'), $('-u'+$upSwitches+'!"'+$deltaArcName+$fExt+'"'), $incParam, $excParam, $addSwitches)

}


function Create-Archive($cfgName, $archType, $a, $p, $o, $i, $x){

    if ([string]::IsNullOrEmpty($archType)) {
        $archType = ""
    }


    if ([string]::IsNullOrEmpty($a) -or -not(Test-Path $a )) {
        $erm = "Archiver program does not exist: '$a'"
        Log "ERROR" $MyInvocation.MyCommand "$erm"
        write-output "ERROR: $erm"
        return 2
    }

    if (!([string]::IsNullOrEmpty($i)) -and -not(Test-Path $i )) {
        $erm = "Include file does not exist: '$i'"
        Log "ERROR" $MyInvocation.MyCommand "$erm"
        write-output "ERROR: $erm"
        return 2
    }

    if (!([string]::IsNullOrEmpty($x)) -and -not(Test-Path $x )) {
        $erm = "Exclude file does not exist: '$x'"
        Log "ERROR" $MyInvocation.MyCommand "$erm"
        write-output "ERROR: $erm"
        return 2
    }

    if ([string]::IsNullOrEmpty($p) -or -not(Test-Path -Path $p )) {
        $erm = "Input folder does not exist: '$p'"
        Log "ERROR" $MyInvocation.MyCommand "$erm"
        write-output "ERROR: $erm"
	    return 2
    }

    if ([string]::IsNullOrEmpty($o) -or -not(Test-Path -Path $o )){
        $erm = "Output folder does not exist: '$o'"
        Log "ERROR" $MyInvocation.MyCommand "$erm"
        write-output "ERROR: $erm"
	    return 2
    }
 
    # base archive name
    $baseArcName = $($o + "\" + $cfgName)

    # archive file containing changed files
    $deltaArcName = $($baseArcName + ' ' + $([datetime]::now.tostring("yyMMddTHHmmss")) + ".chng" )

    switch ($archType.ToUpper()) {
        default {
            $params = Get-Archiver-Params-7Zip $p $o $i $x $baseArcName $deltaArcName
        }
    }
    

    Log "INFO" $MyInvocation.MyCommand "Starting archiver: $a $params"

    # change current folder to input folder so 7zip will include only relative paths
    pushd $p

    # run archiver
    & $a $params

    Log "INFO" $MyInvocation.MyCommand "Archiver exit code = $LASTEXITCODE"
    
    # return to original folder
    popd

    return

}


function Upload-Archive($upType, $upFld, $outFld, $params) {
    $exCode = 0
    $dateFld = Get-Date -Format "yyyy-MM"

    # this scriptblock executes upload
    $uploadScript = {
        param($cmd,$pars,$logF,$bgC)

        Log "INFO" $bgC "Uploading in background using $cmd" $logF;
        & $cmd $pars;
        Log "INFO" $bgC "Uploader exit code = $LASTEXITCODE" $logF;
    }

    # initializatoin script in order Log functionm would be recognized by script block
    $initScr = [scriptblock]::create("function Log {$function:Log}")
   
    # create upload command based on selected uploader type
    # default uploader type is AWS-S3
    switch ($upType.ToUpper()) {
        default {
            $cmd = "aws"
            $pars = @("s3", "mv", $('"'+$outFld+'"'), $($upFld+"/"+$dateFld), "--recursive", "--exclude", '"*"', "--include", '"*.chng.*"') + $params
        }
    }

    #& $cmd $pars
	#start-process $cmd -ArgumentList "$pars" -WorkingDirectory $((Get-Location).path) -NoNewWindow

    # increase number of started background jobs
    $script:bgCount++

    # start background uploader job
    Log "INFO" $MyInvocation.MyCommand "Starting uploader $("bgUpload-$script:bgCount"): $cmd $pars"
    $j = Start-Job -Name $("bgUpload-$script:bgCount") -ScriptBlock $uploadScript -ArgumentList $cmd,$pars,$script:log,$("bgUpload-$script:bgCount") -InitializationScript $initScr
    if ([string]::IsNullOrEmpty($j.ChildJobs[0].Error)){
        Log "INFO" $MyInvocation.MyCommand "Started uploader job $($j.Name)"
        $j.ChildJobs[0].Name = $outFld
        $script:bgUploaders += ($outFld,$j.ChildJobs[0])

    } else {
        Log "ERROR" $MyInvocation.MyCommand "Failed uploading: $($j.ChildJobs[0].Error)"
    }

}

function Process-Section($ini, $s) {

    # check for required parameters
    if (!$ini[$s].ContainsKey("input_folder")) {
        Log "ERROR" $MyInvocation.MyCommand "Input folder not specified in section [$s]"
        return
    }

    # define section specific defaults
    $incFile = $ini[$s]["include"]
    $excFile = $ini[$s]["exclude"]

    # read values from section
    $archType = $ini[$s]["archiver_type"] 
    $archExec = $ini[$s]["archiver_executable"] 
    $outFld = $ini[$s]["output_folder"]
    $upType = $ini[$s]["destination_type"]
    $upFld = $ini[$s]["destination_folder"]
    $upParams = $ini[$s]["uploader_params"]
    $incexcFld = $ini[$s]["incexc_folder"]

    # read values from section [defauls] for not specified parameters
    if ($ini.ContainsKey("defaults")){
        $defs = $ini["defaults"]
    
        if ([string]::IsNullOrEmpty($archType)) 
            {$archType = $defs["archiver_type"]}

        if ([string]::IsNullOrEmpty($archExec)) 
            {$archExec = $defs["archiver_executable"]}

        if ([string]::IsNullOrEmpty($outFld))
            {$outFld = $ini["defaults"]["output_folder"]}

        if ([string]::IsNullOrEmpty($upType)) 
            {$upType = $ini["defaults"]["destination_type"]}

        if ([string]::IsNullOrEmpty($upFld)) 
            {$upFld = $ini["defaults"]["destination_folder"]}

        if ([string]::IsNullOrEmpty($upParams)) 
            {$upParams = $ini["defaults"]["uploader_params"]}

        if ([string]::IsNullOrEmpty($incexcFld)) 
            {$incexcFld = $ini["defaults"]["incexc_folder"]}


    }

    # set global defaults if still needed
    if ([string]::IsNullOrEmpty($archType)) { $archType = "7-zip" } 

    if ([string]::IsNullOrEmpty($archExec)) { $archExec = $defaultArchiver }

    if ([string]::IsNullOrEmpty($outFld)) { 
        Log "ERROR" $MyInvocation.MyCommand "Output folder not specified in section [$s]"
        return
    }

    if ([string]::IsNullOrEmpty($upType)) { $upType = "AWS-S3" }

    if ([string]::IsNullOrEmpty($upFld)) {
        Log "ERROR" $MyInvocation.MyCommand "Destination folder not specified in section [$s]"
        return
    }

    if ([string]::IsNullOrEmpty($upParams)) { $upParams = "" }

    if (![string]::IsNullOrEmpty($incexcFld)) {
        if (![string]::IsNullOrEmpty($incFile)) { $incFile = $incexcFld + "\" + $incFile }
        if (![string]::IsNullOrEmpty($excFile)) { $excFile = $incexcFld + "\" + $excFile }
    }
    
    # create archive
    Create-Archive `
        $s `
        $archType `
        $archExec `
        $ini[$s]["input_folder"] `
        $outFld `
        $incFile `
        $excFile

    # upload archive
    Upload-Archive `
        $upType `
        $upFld `
        $outFld `
        $upParams.Split()

}

# ========= MAIN PROGRAM


if(!(Test-Path -Path $c )){
    write-output "Error: Configuration file does not exist: '$c'"
    exit 2
} 
write-output "Reading configuration from file: '$c'"

$ini = Get-IniContent $c
if ($ini.Keys.Count -eq 0){
    Write-Output "Error: Configuration file does not contain any section."
    exit 2
}

Log "INFO" $MyInvocation.MyCommand "*** START ***"

if ($s -ne ""){
    # section name has been specified
    Log "INFO" $MyInvocation.MyCommand "Processing specified section ['$s']"
    Process-Section $ini $s
    
} else {
    # no section has been explicitly specified -> process all sections
    foreach ($section in $ini.keys) {

        if ($($ini[$section].GetType().Name) -eq "Hashtable") {

            # ignore section DEFAULTS
            if ($section.ToUpper() -ne "DEFAULTS") {
                # ignore if section contains line ignore=yes
                if (($ini[$section]["ignore"] -ne "yes") -or ($ini[$section]["explicit_only"] -ne "yes")) {
                    Log "INFO" $MyInvocation.MyCommand "Processing section ['$section']"
                    Process-Section $ini $section
                   
                } else {
                    Log "INFO" $MyInvocation.MyCommand "Ignoring section ['$section'] as ignore=yes"
                }
            }

        } else {
            Log "WARN" $MyInvocation.MyCommand "Wrong line in config file: $selection"
        }
    }

}


$bgProc = Get-Job -State Running
Log "INFO" $MyInvocation.MyCommand "Waiting for background processess to finish: $($bgProc.Count)"

$waitTime = 15

if ($bgProc.Count -gt 0) {
    do {
        Log "INFO" $MyInvocation.MyCommand "Processes running: $($bgProc.Count)"
        foreach ($u in $bgUploaders){
            if ($u[1].State -eq "Running"){
                Log "INFO" "Uploading $($u[0]): $($u[1].Output[$u[1].Output.Count-1])"
            }
        }
        Write-Host "Waiting $waitTime s"
        Sleep $waitTime
        $bgProc = Get-Job -State Running
    } while ($bgProc.Count -gt 0)
}

Log "INFO" $MyInvocation.MyCommand "*** END ***"