In this post, I am going to share CSOM based PowerShell script to split a large file into multiple chunks of a smaller size and upload it into SharePoint Online site. OneDrive for Business site (ODFB) is built on top of SharePoint Online (SPO), so if you have enough permission on the required user’s OneDrive site, you can use the same script to upload a bulk file into a users’ personal site.
Required details to upload a large file into SharePoint library:
Before proceeding you need to replace the following input parameters in the below script.
$LocalFilePath – Specify your local file path.
$AdminAccount – Provide admin user account.
$AdminPass – Provide admin password.
$SiteURL – SPO site URL or User’s OneDrive site URL.
$TargetFolderName – Folder where you want to upload. Leave empty if you want to upload in the root folder. If you specify the target folder, then the folder should already exist in the Site library root folder.
Upload a large file into SharePoint Library or OneDrive for Business site
The script slices the big file into blocks and uploads the sliced file content chunk by chunk. The script also shows progress status for each file block upload. For smaller size file, this script uses a regular file upload method. If the file size greater than 5 MB, then the script uses the slice by slice upload method. You can change the maximum chunk size limit through parameter $FileChunkSizeInMB.
Function UploadLargeFileInChunk ($Ctx, $TargetFolder, $FilePath, $FileChunkSizeInMB=5) {
# Get the name of the file.
$FileName = [System.IO.Path]::GetFileName($FilePath)
# File object.
[Microsoft.SharePoint.Client.File] $Upload
# Calculate block size in bytes.
$BlockSize = $FileChunkSizeInMB * 1024 * 1024
# Get the size of the file.
$FileSize = (Get-Item $FilePath).length
if ($FileSize -le $BlockSize)
{
# Use regular upload.
$FileStream = New-Object IO.FileStream($FilePath,[System.IO.FileMode]::Open)
$FileCreationInfo = New-Object Microsoft.SharePoint.Client.FileCreationInformation
$FileCreationInfo.Overwrite = $true
$FileCreationInfo.ContentStream = $FileStream
$FileCreationInfo.URL = $FileName
$Upload = $TargetFolder.Files.Add($FileCreationInfo)
$Ctx.Load($Upload)
$Ctx.ExecuteQuery()
return $Upload
}
else
{
# Use chunk by chunk file upload.
# Each block upload requires a unique ID.
$UploadId = [GUID]::NewGuid()
$BytesUploaded = $null
$Fs = $null
$FileSizeInMB = [math]::Round($(($FileSize/1024)/1014),2)
Try
{
$Fs = [System.IO.File]::Open($FilePath, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, [System.IO.FileShare]::ReadWrite)
$br = New-Object System.IO.BinaryReader($Fs)
$buffer = New-Object System.Byte[]($BlockSize)
$lastBuffer = $null
$fileoffset = 0
$totalBytesRead = 0
$bytesRead
$first = $true
$last = $false
# Read data from file system in blocks.
while(($bytesRead = $br.Read($buffer, 0, $buffer.Length)) -gt 0)
{
$totalBytesRead = $totalBytesRead + $bytesRead
# You've reached the end of the file.
if($totalBytesRead -eq $FileSize)
{
$last = $true
# Copy to a new buffer that has the correct size.
$lastBuffer = New-Object System.Byte[]($bytesRead)
[array]::Copy($buffer, 0, $lastBuffer, 0, $bytesRead)
}
try
{
#Show upload file progress
$UploadSizeInBM = [math]::Round($(($totalBytesRead/1024)/1014),2)
Write-Progress -Activity "Uploading file $FileName" -Status "$UploadSizeInBM out of $FileSizeInMB MB uploaded"
}
Catch{}
If($first)
{
$ContentStream = New-Object System.IO.MemoryStream
# Add an empty file.
$fileInfo = New-Object Microsoft.SharePoint.Client.FileCreationInformation
$fileInfo.ContentStream = $ContentStream
$fileInfo.Url = $FileName
$fileInfo.Overwrite = $true
$Upload = $TargetFolder.Files.Add($fileInfo)
$Ctx.Load($Upload)
# Start upload by uploading the first slice.
$s = [System.IO.MemoryStream]::new($buffer)
# Call the start upload method on the first slice.
$BytesUploaded = $Upload.StartUpload($UploadId, $s)
$Ctx.ExecuteQuery()
# fileoffset is the pointer where the next slice will be added.
$fileoffset = $BytesUploaded.Value
# You can only start the upload once.
$first = $false
}
Else
{
# Get a reference to your file.
$Upload = $Ctx.Web.GetFileByServerRelativeUrl($TargetFolder.ServerRelativeUrl + [System.IO.Path]::AltDirectorySeparatorChar + $FileName);
If($last) {
# Is this the last slice of data?
$s = [System.IO.MemoryStream]::new($lastBuffer)
# End sliced upload by calling FinishUpload.
$Upload = $Upload.FinishUpload($UploadId, $fileoffset, $s)
$Ctx.ExecuteQuery()
Write-Host "File upload complete"
# Return the file object for the uploaded file.
return $Upload
}
else
{
$s = [System.IO.MemoryStream]::new($buffer)
# Continue sliced upload.
$BytesUploaded = $Upload.ContinueUpload($UploadId, $fileoffset, $s)
$Ctx.ExecuteQuery()
# Update fileoffset for the next slice.
$fileoffset = $BytesUploaded.Value
}
}
} #// while ((bytesRead = br.Read(buffer, 0, buffer.Length)) > 0)
}
Catch
{
Write-Host $_.Exception.Message -ForegroundColor Red
}
Finally {
if ($Fs -ne $null)
{
$Fs.Dispose()
}
}
}
return $null
}
#Add required references to SharePoint client assembly to use CSOM
[System.Reflection.Assembly]::LoadWithPartialName("Microsoft.SharePoint.Client")
[System.Reflection.Assembly]::LoadWithPartialName("Microsoft.SharePoint.Client.Runtime")
#Specify SharePoint or OneDrive site admin account
$AdminAccount = "admin@<your tenant name>.onmicrosoft.com"
$AdminPass = "admin_password"
#Specify SharePoint Online Site URL or User's OneDrive Site URL
$SiteURL = "https://<your tenant name>.sharepoint.com/sites/site_name"
#$SiteURL = "https://<your tenant name>-my.sharepoint.com/personal/username_domainame_com"
$DocumentLibrary ="Documents"
$TargetFolderName ='' #Leave empty to target root folder
#Connect and Load SharePoint Library and Root Folder
$SecPwd = $(ConvertTo-SecureString $AdminPass -asplaintext -force)
$Ctx = New-Object Microsoft.SharePoint.Client.ClientContext($SiteURL)
$Credentials = New-Object Microsoft.SharePoint.Client.SharePointOnlineCredentials($AdminAccount,$SecPwd)
$Ctx.credentials = $Credentials
$List = $Ctx.Web.Lists.GetByTitle($DocumentLibrary)
$Ctx.Load($List)
$Ctx.Load($List.RootFolder)
$Ctx.ExecuteQuery()
#Setting Target Folder
$TargetFolder = $null;
If($TargetFolderName) {
$TargetFolderRelativeUrl = $List.RootFolder.ServerRelativeUrl+"/"+$TargetFolderName
$TargetFolder = $Ctx.Web.GetFolderByServerRelativeUrl($TargetFolderRelativeUrl)
$Ctx.Load($TargetFolder)
$Ctx.ExecuteQuery()
if(!$TargetFolder.Exists){
Throw "$TargetFolderName - the target folder does not exist in the root folder."
}
} Else {
$TargetFolder = $List.RootFolder
}
#Specify path of local file to upload
$LocalFilePath = "C:\Users\UserName\Documents\MyFiles\my-large-file.xlsx"
#Call upload file function
$UploadFile = UploadLargeFileInChunk -Ctx $Ctx -TargetFolder $TargetFolder -FilePath $LocalFilePath
Advertisement