Mike Swanson
a9bcbc2580
Supply Management migrated (160 files), SPMT launched for 4 remaining folders, Syncro ticket #109277420 opened, SPB license assigned to sysadmin. Script, errors, SP site map, and next steps documented. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
392 lines
14 KiB
PowerShell
392 lines
14 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
Migrates BirthBiologic Datto Workplace files to SharePoint Online.
|
|
|
|
.DESCRIPTION
|
|
Reads files from C:\Users\Public\Desktop\Datto Workplace Server Projects on BB-SERVER
|
|
and uploads them to the corresponding SharePoint document libraries via Microsoft Graph API.
|
|
Supports resume: already-uploaded files are skipped based on a state file.
|
|
|
|
.PARAMETER ClientSecret
|
|
App secret for the ComputerGuru Tenant Admin app (tenant-admin tier).
|
|
Can also be set via env var BB_MIGRATION_SECRET.
|
|
|
|
.PARAMETER Token
|
|
Pre-obtained Graph Bearer token. Use if you already have a valid token.
|
|
|
|
.PARAMETER DattoRoot
|
|
Path to the Datto Workplace root folder.
|
|
Default: C:\Users\Public\Desktop\Datto Workplace Server Projects
|
|
|
|
.PARAMETER OnlyFolder
|
|
Migrate only this top-level Datto folder (e.g. "Donor Services").
|
|
Omit to migrate all folders.
|
|
|
|
.PARAMETER WhatIf
|
|
Dry run. Lists what would be uploaded without transferring anything.
|
|
|
|
.PARAMETER Resume
|
|
Skip files already recorded in the state file. Default: true.
|
|
Pass -Resume:$false to force re-upload everything.
|
|
|
|
.EXAMPLE
|
|
# Dry run on one folder
|
|
.\migrate-datto-to-sharepoint.ps1 -ClientSecret "xxx" -OnlyFolder "Supply Management" -WhatIf
|
|
|
|
.EXAMPLE
|
|
# Real migration, one folder at a time
|
|
.\migrate-datto-to-sharepoint.ps1 -ClientSecret "xxx" -OnlyFolder "Supply Management"
|
|
|
|
.NOTES
|
|
Requires ComputerGuru Tenant Admin app consented in BirthBiologic tenant
|
|
with Sites.ReadWrite.All application permission.
|
|
Run on BB-SERVER where Datto files are locally accessible.
|
|
#>
|
|
|
|
[CmdletBinding()]
|
|
param(
|
|
[string]$ClientSecret = $env:BB_MIGRATION_SECRET,
|
|
[string]$Token = "",
|
|
[string]$DattoRoot = "C:\Users\Public\Desktop\Datto Workplace Server Projects",
|
|
[string]$OnlyFolder = "",
|
|
[switch]$WhatIf,
|
|
[bool]$Resume = $true,
|
|
[switch]$DeltaOnly
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
|
|
|
|
# Constants
|
|
$TENANT_ID = "19a568e8-9e88-413b-9341-cbc224b39145"
|
|
$CLIENT_ID = "709e6eed-0711-4875-9c44-2d3518c47063"
|
|
$GRAPH_ROOT = "https://graph.microsoft.com/v1.0"
|
|
$CHUNK_SIZE = 60 * 1024 * 1024
|
|
$SMALL_FILE = 4 * 1024 * 1024
|
|
$MAX_RETRY = 3
|
|
|
|
$LOG_DIR = "C:\GuruMigration"
|
|
$LOG_FILE = "$LOG_DIR\bb-migration-$(Get-Date -Format 'yyyy-MM-dd').log"
|
|
$STATE_FILE = "$LOG_DIR\bb-migration-state.json"
|
|
|
|
# Site ID map: Datto folder name -> SharePoint site ID
|
|
$SITE_MAP = [ordered]@{
|
|
"Admin" = "birthbiologic.sharepoint.com,1baf65c1-c4b3-4602-9111-1f99ae800023,4a1a8706-42a5-4d72-bb7d-75a846036623"
|
|
"Birth Biologic Activity Reports" = "birthbiologic.sharepoint.com,1baf65c1-c4b3-4602-9111-1f99ae800023,4a1a8706-42a5-4d72-bb7d-75a846036623"
|
|
"Donor Services" = "birthbiologic.sharepoint.com,bcbfa272-dc85-424c-af66-3f14c75ffeb4,8b0975dd-e0ac-4ad0-9921-d7c9c2858865"
|
|
"ITSvcs" = "birthbiologic.sharepoint.com,83b636d9-2e38-4fd8-83eb-431ab1f0067b,8b0975dd-e0ac-4ad0-9921-d7c9c2858865"
|
|
"Quality Department" = "birthbiologic.sharepoint.com,5fd38089-b87f-4472-8d17-2d8bbc2759a2,8b0975dd-e0ac-4ad0-9921-d7c9c2858865"
|
|
"Supply Management" = "birthbiologic.sharepoint.com,4700ecf3-25ba-41b6-918c-9fe620038172,8b0975dd-e0ac-4ad0-9921-d7c9c2858865"
|
|
}
|
|
|
|
# Folders routed into a sub-path at the destination
|
|
$SUBFOLDER_PREFIX = @{
|
|
"Birth Biologic Activity Reports" = "Birth Biologic Activity Reports"
|
|
}
|
|
|
|
# Logging
|
|
if (-not (Test-Path $LOG_DIR)) { New-Item -ItemType Directory -Path $LOG_DIR | Out-Null }
|
|
|
|
function Write-Log {
|
|
param([string]$Msg, [string]$Level = "INFO")
|
|
$line = "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') [$Level] $Msg"
|
|
Write-Host $line
|
|
Add-Content -Path $LOG_FILE -Value $line -Encoding UTF8
|
|
}
|
|
|
|
# State tracking for resume
|
|
$script:state = @{}
|
|
if ($Resume -and (Test-Path $STATE_FILE)) {
|
|
$raw = Get-Content $STATE_FILE -Raw | ConvertFrom-Json
|
|
$raw.PSObject.Properties | ForEach-Object { $script:state[$_.Name] = $_.Value }
|
|
Write-Log "Resume state loaded: $($script:state.Count) files already uploaded"
|
|
}
|
|
|
|
function Save-State {
|
|
$script:state | ConvertTo-Json -Depth 2 | Set-Content $STATE_FILE -Encoding UTF8
|
|
}
|
|
|
|
function Mark-Done {
|
|
param([string]$Key)
|
|
$script:state[$Key] = (Get-Date -Format "o")
|
|
if ($script:state.Count % 25 -eq 0) { Save-State }
|
|
}
|
|
|
|
function Is-Done {
|
|
param([string]$Key)
|
|
return $script:state.ContainsKey($Key)
|
|
}
|
|
|
|
# Token management
|
|
$script:tokenValue = ""
|
|
$script:tokenFetched = 0
|
|
$script:tokenExpiry = 0
|
|
|
|
function Get-Token {
|
|
$now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds()
|
|
if ($script:tokenValue -and ($now + 300) -lt ($script:tokenFetched + $script:tokenExpiry)) {
|
|
return $script:tokenValue
|
|
}
|
|
if ($Token) {
|
|
$script:tokenValue = $Token
|
|
$script:tokenFetched = $now
|
|
$script:tokenExpiry = 3600
|
|
return $Token
|
|
}
|
|
if (-not $ClientSecret) {
|
|
throw "Provide -ClientSecret or -Token, or set env var BB_MIGRATION_SECRET."
|
|
}
|
|
$body = "grant_type=client_credentials&client_id=$CLIENT_ID&client_secret=$([uri]::EscapeDataString($ClientSecret))&scope=https://graph.microsoft.com/.default"
|
|
$resp = Invoke-RestMethod -Method POST `
|
|
-Uri "https://login.microsoftonline.com/$TENANT_ID/oauth2/v2.0/token" `
|
|
-Body $body -ContentType "application/x-www-form-urlencoded"
|
|
$script:tokenValue = $resp.access_token
|
|
$script:tokenFetched = $now
|
|
$script:tokenExpiry = $resp.expires_in
|
|
Write-Log "Token acquired (expires in $($resp.expires_in)s)"
|
|
return $script:tokenValue
|
|
}
|
|
|
|
# Graph API call with retry
|
|
function Invoke-Graph {
|
|
param(
|
|
[string]$Method,
|
|
[string]$Uri,
|
|
[hashtable]$ExtraHeaders = @{},
|
|
[object]$Body = $null,
|
|
[string]$ContentType = "application/json",
|
|
[int]$Attempt = 0
|
|
)
|
|
$tok = Get-Token
|
|
$headers = @{ Authorization = "Bearer $tok" }
|
|
foreach ($k in $ExtraHeaders.Keys) { $headers[$k] = $ExtraHeaders[$k] }
|
|
|
|
$params = @{
|
|
Method = $Method
|
|
Uri = $Uri
|
|
Headers = $headers
|
|
ContentType = $ContentType
|
|
}
|
|
if ($null -ne $Body) { $params.Body = $Body }
|
|
|
|
try {
|
|
return Invoke-RestMethod @params
|
|
} catch {
|
|
$status = 0
|
|
if ($_.Exception.Response) { $status = [int]$_.Exception.Response.StatusCode }
|
|
if (($status -eq 429 -or $status -ge 500) -and $Attempt -lt $MAX_RETRY) {
|
|
$wait = [math]::Pow(2, $Attempt + 1)
|
|
Write-Log "HTTP $status - retry $($Attempt+1)/$MAX_RETRY in ${wait}s" "WARN"
|
|
Start-Sleep -Seconds $wait
|
|
return Invoke-Graph -Method $Method -Uri $Uri -ExtraHeaders $ExtraHeaders `
|
|
-Body $Body -ContentType $ContentType -Attempt ($Attempt + 1)
|
|
}
|
|
throw
|
|
}
|
|
}
|
|
|
|
# Encode each path segment individually
|
|
function Encode-Path {
|
|
param([string]$RelPath)
|
|
return ($RelPath.Split('/') | ForEach-Object { [uri]::EscapeDataString($_) }) -join '/'
|
|
}
|
|
|
|
# Upload small file (<= 4 MB)
|
|
function Upload-SmallFile {
|
|
param([string]$SiteId, [string]$RemotePath, [string]$LocalPath)
|
|
$encoded = Encode-Path $RemotePath
|
|
$uri = "${GRAPH_ROOT}/sites/${SiteId}/drive/root:/${encoded}:/content"
|
|
$bytes = [System.IO.File]::ReadAllBytes($LocalPath)
|
|
return Invoke-Graph -Method PUT -Uri $uri -Body $bytes -ContentType "application/octet-stream"
|
|
}
|
|
|
|
# Upload large file via chunked upload session
|
|
function Upload-LargeFile {
|
|
param([string]$SiteId, [string]$RemotePath, [string]$LocalPath)
|
|
|
|
$fileSize = (Get-Item $LocalPath).Length
|
|
$encoded = Encode-Path $RemotePath
|
|
|
|
# Delete any existing/partial item first so upload session creation doesn't 409
|
|
$deleteUri = "${GRAPH_ROOT}/sites/${SiteId}/drive/root:/${encoded}"
|
|
try { Invoke-Graph -Method DELETE -Uri $deleteUri | Out-Null } catch {}
|
|
|
|
$sessionUri = "${GRAPH_ROOT}/sites/${SiteId}/drive/root:/${encoded}:/createUploadSession"
|
|
$sessionBody = '{"item":{"@microsoft.graph.conflictBehavior":"replace"}}'
|
|
$session = Invoke-Graph -Method POST -Uri $sessionUri -Body $sessionBody
|
|
$uploadUrl = $session.uploadUrl
|
|
|
|
$stream = [System.IO.File]::OpenRead($LocalPath)
|
|
$buf = New-Object byte[] $CHUNK_SIZE
|
|
$offset = 0L
|
|
$resp = $null
|
|
|
|
try {
|
|
while ($offset -lt $fileSize) {
|
|
$read = $stream.Read($buf, 0, $CHUNK_SIZE)
|
|
if ($read -eq 0) { break }
|
|
$chunk = if ($read -eq $CHUNK_SIZE) { $buf } else { $buf[0..($read - 1)] }
|
|
$end = $offset + $read - 1
|
|
$pct = [int]($offset * 100 / $fileSize)
|
|
Write-Host "`r $pct% ($([math]::Round($offset/1MB,0)) MB / $([math]::Round($fileSize/1MB,0)) MB) " -NoNewline
|
|
|
|
$attempt = 0
|
|
$ok = $false
|
|
while (-not $ok) {
|
|
try {
|
|
$resp = Invoke-RestMethod -Method PUT -Uri $uploadUrl `
|
|
-Headers @{ "Content-Range" = "bytes $offset-$end/$fileSize" } `
|
|
-Body $chunk -ContentType "application/octet-stream"
|
|
$ok = $true
|
|
} catch {
|
|
$attempt++
|
|
if ($attempt -ge $MAX_RETRY) { throw }
|
|
Start-Sleep -Seconds ([math]::Pow(2, $attempt))
|
|
}
|
|
}
|
|
$offset += $read
|
|
}
|
|
} catch {
|
|
# Cancel the upload session so outer retries can start a fresh one
|
|
try { Invoke-RestMethod -Method DELETE -Uri $uploadUrl -ErrorAction SilentlyContinue } catch {}
|
|
throw
|
|
} finally {
|
|
$stream.Dispose()
|
|
Write-Host ""
|
|
}
|
|
return $resp
|
|
}
|
|
|
|
function Upload-File {
|
|
param([string]$SiteId, [string]$RemotePath, [string]$LocalPath)
|
|
$size = (Get-Item $LocalPath).Length
|
|
if ($size -le $SMALL_FILE) {
|
|
return Upload-SmallFile -SiteId $SiteId -RemotePath $RemotePath -LocalPath $LocalPath
|
|
} else {
|
|
return Upload-LargeFile -SiteId $SiteId -RemotePath $RemotePath -LocalPath $LocalPath
|
|
}
|
|
}
|
|
|
|
# Migrate one Datto top-level folder
|
|
function Migrate-Folder {
|
|
param([string]$DattoFolder, [string]$SiteId, [string]$SitePrefix)
|
|
|
|
$srcRoot = Join-Path $DattoRoot $DattoFolder
|
|
if (-not (Test-Path $srcRoot)) {
|
|
Write-Log "Source not found, skipping: $srcRoot" "WARN"
|
|
return $null
|
|
}
|
|
|
|
$files = Get-ChildItem -Path $srcRoot -Recurse -File
|
|
$total = $files.Count
|
|
$done = 0
|
|
$skip = 0
|
|
$fail = 0
|
|
|
|
Write-Log "[$DattoFolder] $total files to process..."
|
|
|
|
foreach ($file in $files) {
|
|
$relPath = $file.FullName.Substring($srcRoot.Length).TrimStart('\').Replace('\', '/')
|
|
$remotePath = if ($SitePrefix) { "$SitePrefix/$relPath" } else { $relPath }
|
|
$stateKey = "$DattoFolder/$relPath"
|
|
|
|
if ($Resume -and (Is-Done $stateKey)) {
|
|
$skip++
|
|
continue
|
|
}
|
|
|
|
$sizeMB = [math]::Round($file.Length / 1MB, 2)
|
|
Write-Log " [$($done + $skip + $fail + 1)/$total] $relPath ($sizeMB MB)"
|
|
|
|
if ($DeltaOnly) {
|
|
# Check if SharePoint already has this file and it's current
|
|
$encoded = Encode-Path $remotePath
|
|
$checkUri = "${GRAPH_ROOT}/sites/${SiteId}/drive/root:/${encoded}?select=lastModifiedDateTime,size"
|
|
try {
|
|
$spItem = Invoke-Graph -Method GET -Uri $checkUri
|
|
$spDate = [datetime]$spItem.lastModifiedDateTime
|
|
$localDate = $file.LastWriteTimeUtc
|
|
if ($spItem.size -gt 0 -and $spDate -ge $localDate) {
|
|
$skip++
|
|
continue
|
|
}
|
|
Write-Log " [DELTA] $relPath (SP: $($spDate.ToString('yyyy-MM-dd')) / Local: $($localDate.ToString('yyyy-MM-dd')))"
|
|
} catch {
|
|
# Not found in SP — upload it
|
|
}
|
|
}
|
|
|
|
if ($WhatIf) {
|
|
Write-Log " [WHATIF] -> $remotePath"
|
|
$done++
|
|
continue
|
|
}
|
|
|
|
$attempt = 0
|
|
$ok = $false
|
|
while ($attempt -lt $MAX_RETRY -and -not $ok) {
|
|
try {
|
|
Upload-File -SiteId $SiteId -RemotePath $remotePath -LocalPath $file.FullName | Out-Null
|
|
Mark-Done $stateKey
|
|
$done++
|
|
$ok = $true
|
|
} catch {
|
|
$attempt++
|
|
$errMsg = $_.Exception.Message
|
|
if ($attempt -ge $MAX_RETRY) {
|
|
Write-Log " [FAIL] $relPath - $errMsg" "ERROR"
|
|
$fail++
|
|
} else {
|
|
Write-Log " [RETRY $attempt] $relPath - $errMsg" "WARN"
|
|
Start-Sleep -Seconds ([math]::Pow(2, $attempt))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Save-State
|
|
Write-Log "[$DattoFolder] Done: $done uploaded, $skip skipped, $fail failed / $total total"
|
|
return [pscustomobject]@{ Folder = $DattoFolder; Uploaded = $done; Skipped = $skip; Failed = $fail; Total = $total }
|
|
}
|
|
|
|
# Main
|
|
Write-Log "=== BirthBiologic Datto -> SharePoint Migration ==="
|
|
Write-Log "Source: $DattoRoot"
|
|
Write-Log "WhatIf=$WhatIf | Resume=$Resume | DeltaOnly=$DeltaOnly | OnlyFolder=$(if ($OnlyFolder) { $OnlyFolder } else { '(all)' })"
|
|
|
|
if (-not (Test-Path $DattoRoot)) {
|
|
Write-Log "ERROR: Datto root not found: $DattoRoot" "ERROR"
|
|
exit 1
|
|
}
|
|
|
|
try { $null = Get-Token; Write-Log "Auth OK" }
|
|
catch { Write-Log "Auth FAILED: $_" "ERROR"; exit 1 }
|
|
|
|
$results = @()
|
|
$startTime = Get-Date
|
|
|
|
foreach ($folderName in $SITE_MAP.Keys) {
|
|
if ($OnlyFolder -and $folderName -ne $OnlyFolder) { continue }
|
|
$siteId = $SITE_MAP[$folderName]
|
|
$prefix = if ($SUBFOLDER_PREFIX.ContainsKey($folderName)) { $SUBFOLDER_PREFIX[$folderName] } else { "" }
|
|
$result = Migrate-Folder -DattoFolder $folderName -SiteId $siteId -SitePrefix $prefix
|
|
if ($result) { $results += $result }
|
|
}
|
|
|
|
Save-State
|
|
|
|
$elapsed = (Get-Date) - $startTime
|
|
$totalUp = ($results | Measure-Object -Property Uploaded -Sum).Sum
|
|
$totalSkip = ($results | Measure-Object -Property Skipped -Sum).Sum
|
|
$totalFail = ($results | Measure-Object -Property Failed -Sum).Sum
|
|
|
|
Write-Log ""
|
|
Write-Log "=== Summary ==="
|
|
Write-Log "Elapsed: $([math]::Round($elapsed.TotalMinutes, 1)) min"
|
|
Write-Log "Uploaded: $totalUp Skipped: $totalSkip Failed: $totalFail"
|
|
foreach ($r in $results) {
|
|
Write-Log " $($r.Folder): $($r.Uploaded) up / $($r.Skipped) skip / $($r.Failed) fail / $($r.Total) total"
|
|
}
|
|
if ($totalFail -gt 0) { Write-Log "Some failures - check log: $LOG_FILE" "WARN"; exit 1 }
|
|
Write-Log "=== Done ==="
|