AD2 session 2026-03-27/28/29: Test datasheet pipeline rebuild
- Built exact-match TXT formatter from QuickBASIC source (SCM5B, 8B, DSCA, DSCT, SCM7B) - Spec parser for 10 binary DAT files (1470+ models) - Work order report importer (33K WOs, 63K test lines) - On-demand PDF generation, styled HTML view - Archived 500K pre-2026 For_Web files into year subfolders - Created domain service account (INTRANET\svc_testdatadb) - Generated 73/73 Quatronix customer datasheets - Added STAGE + Reports auto-import to sync script Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
sysadmin
264
Test Datasheets/TestDataSheetUploader/Module1.vb
Normal file
264
Test Datasheets/TestDataSheetUploader/Module1.vb
Normal file
@@ -0,0 +1,264 @@
|
||||
Module Module1
|
||||
|
||||
Private processLog As System.Text.StringBuilder
|
||||
|
||||
Sub Main()
|
||||
processLog = New System.Text.StringBuilder()
|
||||
Dim appSettings As New Configuration.AppSettingsReader
|
||||
|
||||
|
||||
'for testing only
|
||||
'UploadFilesInDirectory("C:\Users\hoffm\Documents\Customer Folders\Dataforth\product lists", "TestFolder")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
'CopyInventoryFilesToStagingFolder()
|
||||
'SyncFolder("InventoryDataFolder")
|
||||
|
||||
SyncFolder("TestDataSheet")
|
||||
|
||||
|
||||
Dim logFilename As String = My.Application.Info.DirectoryPath & "\processLog." & Now.ToString("yyyy.MM.dd.HH.mm.ss") & ".txt"
|
||||
My.Computer.FileSystem.WriteAllText(logFilename, processLog.ToString, False)
|
||||
|
||||
|
||||
End Sub
|
||||
|
||||
Private Sub AddToLog(ByVal message As String, ByVal Echo As Boolean)
|
||||
processLog.Append(Now.ToString("MM/dd/YYYY HH:mm:ss | ") & message & vbCrLf)
|
||||
If Echo Then
|
||||
Console.WriteLine(message)
|
||||
End If
|
||||
End Sub
|
||||
|
||||
|
||||
Public Sub CopyInventoryFilesToStagingFolder()
|
||||
|
||||
Dim appSettings As New Configuration.AppSettingsReader
|
||||
Dim stagingFolderPath As String = appSettings.GetValue("InventoryDataStagingFolder", GetType(String))
|
||||
Dim srcFolderPath As String = appSettings.GetValue("InventoryDataSourceFolder", GetType(String))
|
||||
Dim filenamesStr As String = appSettings.GetValue("InventoryDataFiles", GetType(String))
|
||||
|
||||
Dim filenames() As String = filenamesStr.Split(",")
|
||||
|
||||
For Each thisFilename As String In filenames
|
||||
Dim src As String = srcFolderPath & "\" & thisFilename
|
||||
Dim dest As String = stagingFolderPath & "\" & thisFilename
|
||||
My.Computer.FileSystem.CopyFile(src, dest, True)
|
||||
Next
|
||||
|
||||
|
||||
End Sub
|
||||
|
||||
|
||||
Public Sub UploadFilesInDirectory(ByVal DirectoryPath As String, ByVal UploadType As String)
|
||||
|
||||
|
||||
|
||||
Dim appSettings As New Configuration.AppSettingsReader
|
||||
Dim username As String = appSettings.GetValue("ServiceUsername", GetType(String))
|
||||
Dim password As String = appSettings.GetValue("ServicePassword", GetType(String))
|
||||
Dim serviceCredentials As New Net.NetworkCredential(username, password)
|
||||
Dim url As String = appSettings.GetValue("UploaderServiceURL", GetType(String))
|
||||
|
||||
|
||||
|
||||
For Each thisFile As String In My.Computer.FileSystem.GetFiles(DirectoryPath)
|
||||
|
||||
Dim formParameters As New System.Collections.Specialized.NameValueCollection
|
||||
formParameters.Add("UploadType", UploadType)
|
||||
|
||||
Dim logMessage As New String("")
|
||||
Console.WriteLine("Uploading: " & thisFile)
|
||||
If HTTPUploader.UploadFile(url, serviceCredentials, thisFile, formParameters, logMessage) Then
|
||||
'this video is now staged
|
||||
AddToLog("Uploaded " & thisFile, True)
|
||||
|
||||
My.Computer.FileSystem.DeleteFile(thisFile)
|
||||
Else
|
||||
|
||||
AddToLog("Upload FAILED for " & thisFile, True)
|
||||
AddToLog("Upload FAIL MESSAGE: " & logMessage, False)
|
||||
Console.WriteLine("ERROR: " & logMessage)
|
||||
End If
|
||||
|
||||
|
||||
Next
|
||||
|
||||
|
||||
End Sub
|
||||
|
||||
Public Sub SyncFolder(ByVal FolderAlias As String)
|
||||
Try
|
||||
AddToLog("SyncFolder(" & FolderAlias & ") started.", True)
|
||||
'get manifest of files that are on server mirror
|
||||
Dim appSettings As New Configuration.AppSettingsReader
|
||||
Dim username As String = appSettings.GetValue("ServiceUsername", GetType(String))
|
||||
Dim password As String = appSettings.GetValue("ServicePassword", GetType(String))
|
||||
Dim serviceCredentials As New Net.NetworkCredential(username, password)
|
||||
Dim manifestServiceUrl As String = appSettings.GetValue("DirectoryManifestServiceURL", GetType(String))
|
||||
Dim uploadServiceUrl As String = appSettings.GetValue("UploaderServiceURL", GetType(String))
|
||||
Dim deleteServiceUrl As String = appSettings.GetValue("DeleteFileServiceURL", GetType(String))
|
||||
Dim testMode As Boolean = appSettings.GetValue("TestMode", GetType(Boolean))
|
||||
|
||||
'Create an XML file to post with the request
|
||||
Dim requestDS As New DataSet("Request")
|
||||
Dim requestDT As New DataTable("RequestData")
|
||||
requestDT.Columns.Add("SyncPathAlias", GetType(String))
|
||||
requestDS.Tables.Add(requestDT)
|
||||
Dim requestDR As DataRow = requestDT.NewRow
|
||||
requestDR("SyncPathAlias") = FolderAlias
|
||||
requestDT.Rows.Add(requestDR)
|
||||
|
||||
'Get Dataset from XML response
|
||||
Dim manifestXML As String = CSFramework_Utilities.XMLData.DownloadXMLWithDatasetPost(manifestServiceUrl, requestDS, serviceCredentials)
|
||||
|
||||
My.Computer.FileSystem.WriteAllText(My.Application.Info.DirectoryPath & "\manifest.xml", manifestXML, False)
|
||||
|
||||
Dim ds As DataSet = CSFramework_Utilities.XMLData.DatasetFromXML(manifestXML)
|
||||
|
||||
AddToLog("Manifest downloaded with " & ds.Tables("ContentFile").Rows.Count & " files.", True)
|
||||
|
||||
'Build a list of files that should be uploaded
|
||||
Dim filesToUpload As New List(Of String)
|
||||
|
||||
Dim localFolderPath As String = ""
|
||||
Select Case FolderAlias
|
||||
Case "TestDataSheet"
|
||||
localFolderPath = appSettings.GetValue("TestDataSheetPath", GetType(String))
|
||||
Case "TestFolder"
|
||||
localFolderPath = appSettings.GetValue("TestFolderPath", GetType(String))
|
||||
Case "InventoryDataFolder"
|
||||
localFolderPath = appSettings.GetValue("InventoryDataStagingFolder", GetType(String))
|
||||
End Select
|
||||
|
||||
Dim discoveryFileCount As Integer = 0
|
||||
Dim discoveryFileEchoCount As Integer = 1000
|
||||
|
||||
Dim sourceFolder As New System.IO.DirectoryInfo(localFolderPath)
|
||||
AddToLog("Accessing local folder: " & localFolderPath, True)
|
||||
|
||||
For Each thisLocalFileInfo As System.IO.FileInfo In sourceFolder.GetFiles
|
||||
discoveryFileCount += 1
|
||||
If discoveryFileCount = 1 Then
|
||||
AddToLog("Found at least 1 file in local folder", True)
|
||||
End If
|
||||
If discoveryFileCount = discoveryFileEchoCount Then
|
||||
AddToLog("Discovery file count: " & discoveryFileEchoCount, True)
|
||||
discoveryFileEchoCount += 1000
|
||||
End If
|
||||
'Dim thisLocalFileInfo As New System.IO.FileInfo(thisLocalFile)
|
||||
|
||||
|
||||
If thisLocalFileInfo.LastWriteTimeUtc.Year = Now.Year Then
|
||||
Dim foundMatchingFileOnServer As Boolean = False
|
||||
If ds.Tables.Contains("ContentFile") Then
|
||||
For Each dr As DataRow In ds.Tables("ContentFile").Rows
|
||||
If dr("Filename").ToString = thisLocalFileInfo.Name Then
|
||||
Dim filesize As Integer = dr("Filesize")
|
||||
Dim datelastupdated As Date = dr("DateLastUpdated")
|
||||
datelastupdated = datelastupdated.ToUniversalTime
|
||||
If filesize = thisLocalFileInfo.Length Then
|
||||
If thisLocalFileInfo.LastWriteTimeUtc.ToString("yyyy.MM.dd.HH.mm.ss") = datelastupdated.ToString("yyyy.MM.dd.HH.mm.ss") Then
|
||||
foundMatchingFileOnServer = True
|
||||
Exit For
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
If Not foundMatchingFileOnServer Then
|
||||
filesToUpload.Add(thisLocalFileInfo.FullName)
|
||||
AddToLog("Requesting upload: " & thisLocalFileInfo.Name, True)
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
|
||||
AddToLog("Found " & filesToUpload.Count & " files that need to be uploaded.", True)
|
||||
|
||||
'Upload Files to Server
|
||||
If Not testMode Then
|
||||
For Each thisFile As String In filesToUpload
|
||||
Dim thisLocalFileInfo As New System.IO.FileInfo(thisFile)
|
||||
Dim formParameters As New System.Collections.Specialized.NameValueCollection
|
||||
formParameters.Add("UploadType", FolderAlias)
|
||||
'Request("LastUpdatedDate")
|
||||
formParameters.Add("LastUpdatedDate", thisLocalFileInfo.LastWriteTimeUtc.ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ"))
|
||||
'2015-12-11T17:38:05.9190462Z
|
||||
|
||||
Dim logMessage As New String("")
|
||||
Console.WriteLine("Uploading: " & thisFile)
|
||||
If HTTPUploader.UploadFile(uploadServiceUrl, serviceCredentials, thisFile, formParameters, logMessage) Then
|
||||
'this file is now staged
|
||||
AddToLog("Uploaded file: " & thisFile, True)
|
||||
|
||||
'My.Computer.FileSystem.DeleteFile(thisFile)
|
||||
Else
|
||||
|
||||
AddToLog("Upload FAILED for " & thisFile, True)
|
||||
AddToLog("Upload FAIL MESSAGE: " & logMessage, False)
|
||||
Console.WriteLine("ERROR: " & logMessage)
|
||||
End If
|
||||
|
||||
|
||||
Next
|
||||
End If
|
||||
|
||||
|
||||
'Create dataset for DeleteFile service
|
||||
Dim deleteDS As New DataSet("DeleteRequestData")
|
||||
|
||||
|
||||
'add request data
|
||||
Dim deleteRD As New DataTable("RequestData")
|
||||
deleteRD.Columns.Add("SyncPathAlias", GetType(String))
|
||||
Dim deleteDataDR As DataRow = deleteRD.NewRow
|
||||
deleteDataDR("SyncPathAlias") = FolderAlias
|
||||
deleteRD.Rows.Add(deleteDataDR)
|
||||
deleteDS.Tables.Add(deleteRD)
|
||||
|
||||
|
||||
'add filenames
|
||||
Dim deleteDT As New DataTable("Filenames")
|
||||
deleteDT.Columns.Add("Filename", GetType(String))
|
||||
deleteDS.Tables.Add(deleteDT)
|
||||
|
||||
If ds.Tables.Contains("ContentFile") Then
|
||||
For Each dr As DataRow In ds.Tables("ContentFile").Rows
|
||||
Dim localFileName As String = localFolderPath & "\" & dr("Filename").ToString
|
||||
If Not My.Computer.FileSystem.FileExists(localFileName) Then
|
||||
|
||||
Dim deleteDR As DataRow = deleteDT.NewRow
|
||||
deleteDR("Filename") = dr("Filename").ToString
|
||||
deleteDT.Rows.Add(deleteDR)
|
||||
|
||||
AddToLog("Requested " & FolderAlias & " Delete: " & dr("Filename").ToString, True)
|
||||
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
|
||||
AddToLog("Found " & deleteDT.Rows.Count & " files to delete.", True)
|
||||
|
||||
|
||||
'Delete obsolete files that were detected
|
||||
If Not testMode Then
|
||||
If deleteDT.Rows.Count > 0 Then
|
||||
Dim deleteResponseXML As String = CSFramework_Utilities.XMLData.DownloadXMLWithDatasetPost(deleteServiceUrl, deleteDS, serviceCredentials)
|
||||
AddToLog("Obsolete " & FolderAlias & " files deleted.", True)
|
||||
Else
|
||||
AddToLog("No obsolete " & FolderAlias & " files were detected.", True)
|
||||
End If
|
||||
End If
|
||||
|
||||
Catch ex As Exception
|
||||
AddToLog("Exception in SyncFolder: " & ex.ToString, True)
|
||||
|
||||
|
||||
End Try
|
||||
End Sub
|
||||
|
||||
|
||||
End Module
|
||||
Reference in New Issue
Block a user