In this post, we are going to share CSOM based PowerShell script to recursively read files from nested local folders and upload files into the SharePoint Online site with the same folder structure. This script also reads a large file in chunks from a local folder and uploads the file chunk by chunk.
OneDrive for Business site (ODFB) is built on top of SharePoint Online (SPO), so if you have enough permission on the required user’s OneDrive site, then you can use the same script to upload files and folder structure into a users’ personal site.
Required details to upload files and folders into SharePoint library:
Before proceeding you need to replace the following input parameters in the below script.
$LocalFolder – Specify your local folder path.
$AdminAccount – Provide admin user account.
$AdminPass – Provide admin password.
$SiteURL – SPO site URL or User’s OneDrive site URL.
$TargetFolderName – Folder where you want to upload. Leave empty if you want to upload in the root folder. If you specify the target folder, then the folder should already exist in the Site library root folder.
Upload large files from local folder and subfolders with folder structure
For smaller size files, this script uses a regular file upload method. If the file size greater than 5 MB, then the script split the file into blocks and uploads the sliced file content chunk by chunk. You can change the maximum chunk size limit through the parameter $FileChunkSizeInMB.
Function UploadLargeFileInChunk ($Ctx, $TargetFolder, $TargetFileURL, $FilePath, $FileChunkSizeInMB=5) {
# Get the name of the file.
$FileName = [System.IO.Path]::GetFileName($FilePath)
# File object.
[Microsoft.SharePoint.Client.File] $Upload
# Calculate block size in bytes.
$BlockSize = $FileChunkSizeInMB * 1024 * 1024
# Get the size of the file.
$FileSize = (Get-Item $FilePath).length
if ($FileSize -le $BlockSize)
{
# Use regular upload.
$FileStream = New-Object IO.FileStream($FilePath,[System.IO.FileMode]::Open)
$FileCreationInfo = New-Object Microsoft.SharePoint.Client.FileCreationInformation
$FileCreationInfo.Overwrite = $true
$FileCreationInfo.ContentStream = $FileStream
$FileCreationInfo.URL = $TargetFileURL
$Upload = $TargetFolder.Files.Add($FileCreationInfo)
$Ctx.Load($Upload)
$Ctx.ExecuteQuery()
return $Upload
}
else
{
# Use chunk by chunk file upload.
# Each block upload requires a unique ID.
$UploadId = [GUID]::NewGuid()
$BytesUploaded = $null
$Fs = $null
$FileSizeInMB = [math]::Round($(($FileSize/1024)/1014),2)
Try
{
$Fs = [System.IO.File]::Open($FilePath, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, [System.IO.FileShare]::ReadWrite)
$br = New-Object System.IO.BinaryReader($Fs)
$buffer = New-Object System.Byte[]($BlockSize)
$lastBuffer = $null
$fileoffset = 0
$totalBytesRead = 0
$bytesRead
$first = $true
$last = $false
# Read data from file system in blocks.
while(($bytesRead = $br.Read($buffer, 0, $buffer.Length)) -gt 0)
{
$totalBytesRead = $totalBytesRead + $bytesRead
# You've reached the end of the file.
if($totalBytesRead -eq $FileSize)
{
$last = $true
# Copy to a new buffer that has the correct size.
$lastBuffer = New-Object System.Byte[]($bytesRead)
[array]::Copy($buffer, 0, $lastBuffer, 0, $bytesRead)
}
try
{
#Show upload file progress
$UploadSizeInBM = [math]::Round($(($totalBytesRead/1024)/1014),2)
Write-Progress -Activity "Uploading file $FileName" -Status "$UploadSizeInBM out of $FileSizeInMB MB uploaded"
}
Catch{}
If($first)
{
$ContentStream = New-Object System.IO.MemoryStream
# Add an empty file.
$fileInfo = New-Object Microsoft.SharePoint.Client.FileCreationInformation
$fileInfo.ContentStream = $ContentStream
$fileInfo.Url = $TargetFileURL
$fileInfo.Overwrite = $true
$Upload = $TargetFolder.Files.Add($fileInfo)
$Ctx.Load($Upload)
# Start upload by uploading the first slice.
$s = [System.IO.MemoryStream]::new($buffer)
# Call the start upload method on the first slice.
$BytesUploaded = $Upload.StartUpload($UploadId, $s)
$Ctx.ExecuteQuery()
# fileoffset is the pointer where the next slice will be added.
$fileoffset = $BytesUploaded.Value
# You can only start the upload once.
$first = $false
}
Else
{
# Get a reference to your file.
$Upload = $Ctx.Web.GetFileByServerRelativeUrl($TargetFileURL);
If($last) {
# Is this the last slice of data?
$s = [System.IO.MemoryStream]::new($lastBuffer)
# End sliced upload by calling FinishUpload.
$Upload = $Upload.FinishUpload($UploadId, $fileoffset, $s)
$Ctx.ExecuteQuery()
# Large file upload complete. Return the file object for the uploaded file.
return $Upload
}
else
{
$s = [System.IO.MemoryStream]::new($buffer)
# Continue sliced upload.
$BytesUploaded = $Upload.ContinueUpload($UploadId, $fileoffset, $s)
$Ctx.ExecuteQuery()
# Update fileoffset for the next slice.
$fileoffset = $BytesUploaded.Value
}
}
} #// while ((bytesRead = br.Read(buffer, 0, buffer.Length)) > 0)
}
Catch
{
Write-Host $_.Exception.Message -ForegroundColor Red
}
Finally {
if ($Fs -ne $null)
{
$Fs.Dispose()
}
}
}
return $null
}
#Add required references to SharePoint client assembly to use CSOM
[System.Reflection.Assembly]::LoadWithPartialName("Microsoft.SharePoint.Client")
[System.Reflection.Assembly]::LoadWithPartialName("Microsoft.SharePoint.Client.Runtime")
#Specify local folder path
$LocalFolder = "C:\Users\UserName\Documents\MyFiles"
#Specify SharePoint or OneDrive site admin account
$AdminAccount = "admin@<your tenant name>.onmicrosoft.com"
$AdminPass = "admin_password"
#Specify SharePoint Online Site URL or User's OneDrive Site URL
$SiteURL = "https://<your tenant name>.sharepoint.com/sites/site_name"
#$SiteURL = "https://<your tenant name>-my.sharepoint.com/personal/username_domainame_com"
$DocumentLibrary ="Documents"
$TargetFolderName ='' #Leave empty to target root folder
#Connect and Load SharePoint Library and Root Folder
$SecPwd = $(ConvertTo-SecureString $AdminPass -asplaintext -force)
$Ctx = New-Object Microsoft.SharePoint.Client.ClientContext($SiteURL)
$Credentials = New-Object Microsoft.SharePoint.Client.SharePointOnlineCredentials($AdminAccount,$SecPwd)
$Ctx.credentials = $Credentials
$List = $Ctx.Web.Lists.GetByTitle($DocumentLibrary)
$Ctx.Load($List)
$Ctx.Load($List.RootFolder)
$Ctx.ExecuteQuery()
# Setting Target Folder
$TargetFolder = $null;
$TargetFolderRelativeUrl;
If($TargetFolderName) {
$TargetFolderRelativeUrl = $List.RootFolder.ServerRelativeUrl+"/"+$TargetFolderName
$TargetFolder = $Ctx.Web.GetFolderByServerRelativeUrl($TargetFolderRelativeUrl)
$Ctx.Load($TargetFolder)
$Ctx.ExecuteQuery()
if(!$TargetFolder.Exists){
Throw "$TargetFolderName - the target folder doest not exist in the root folder."
}
} Else {
$TargetFolder = $List.RootFolder
$TargetFolderRelativeUrl = $TargetFolder.ServerRelativeUrl
}
# Get folders and sub folders from source location
$folders= @()
foreach ($file in (Get-ChildItem -Recurse -Path $LocalFolder -Attributes Directory))
{
$folders +=($file.FullName).replace($LocalFolder+"\",'')
}
# Create folders and sub-folders in the destination location
Write-Progress -activity "Creating folder structure in OneDrive" -status "Creating Folder"
foreach ($folder in $folders)
{
$subfolder_names = $folder.split("\")
$subfolder = $TargetFolder.Folders.Add($subfolder_names[0])
$Ctx.Load($subfolder)
$Ctx.ExecuteQuery()
for ($i = 1; $i -le ($subfolder_names.Count-1) ; $i++)
{
$subfolder = $subfolder.folders.Add($subfolder_names[$i])
$Ctx.Load($subfolder)
$Ctx.ExecuteQuery()
}
}
#Read all files recursively from the local folder and upload into corresponding SharePoint or OneDrive folder.
$i = 1
$Files = (Dir $LocalFolder -File -Recurse)
$TotoalFiles = $Files.Length
ForEach($File in $Files)
{
Try
{
Write-Progress -activity "Uplading $File" -status "$i out of $TotoalFiles completed"
# Target file URL with same folder structure
$TargetFileURL = $TargetFolderRelativeUrl +(($file.FullName).Replace($LocalFolder,'')).Replace("\","/")
$UploadFile = UploadLargeFileInChunk -Ctx $Ctx -TargetFolder $TargetFolder -TargetFileURL $TargetFileURL -FilePath $File.FullName
}
catch {
Write-Host $_.Exception.Message -Forground "Red"
}
$i++
}
Advertisement