1+
2+ // =================================================================== //
3+ // This is a tool to split the native .so file of linux gpu library //
4+ // =================================================================== //
5+
6+ using System . Security . Cryptography ;
7+
8+ string filename = "libtensorflow.so" ;
9+ int count = 5 ;
10+ SplitFile ( filename , count ) ;
11+
12+ static void SplitFile ( string filename , int count )
13+ {
14+ // 打开读取二进制文件的文件流
15+ using ( FileStream input = new FileStream ( filename , FileMode . Open , FileAccess . Read ) )
16+ {
17+ long filesize = new FileInfo ( filename ) . Length ; // 获取文件大小
18+ long fragmentSize = ( long ) ( filesize / count + 1 ) ; // 计算每个分片的大小
19+
20+ byte [ ] buffer = new byte [ fragmentSize ] ; // 设置缓冲区大小
21+ int bytesRead ; // 存储读取长度
22+ int fragmentIndex = 1 ; // 分片计数器
23+
24+ // 使用循环遍历分片并写入相应的文件
25+ while ( ( bytesRead = input . Read ( buffer , 0 , buffer . Length ) ) > 0 )
26+ {
27+ string outputFileName = $ "{ filename } .fragment{ fragmentIndex ++ } ";
28+ using ( FileStream output = new FileStream ( outputFileName , FileMode . Create , FileAccess . Write ) )
29+ {
30+ output . Write ( buffer , 0 , bytesRead ) ;
31+ }
32+ }
33+
34+ // 计算整个文件的 SHA-256 哈希值并写入 .sha 文件
35+ using ( SHA256 sha256Hash = SHA256 . Create ( ) )
36+ {
37+ input . Seek ( 0 , SeekOrigin . Begin ) ;
38+ byte [ ] hashValue = sha256Hash . ComputeHash ( input ) ;
39+
40+ string shaFileName = $ "{ filename } .sha";
41+ using ( StreamWriter writer = new StreamWriter ( shaFileName , false ) )
42+ {
43+ writer . Write ( BitConverter . ToString ( hashValue ) . Replace ( "-" , "" ) ) ;
44+ }
45+ }
46+ }
47+ }
48+
49+ // Resume the file from fregments. Thanks for the code in TorchSharp!
50+ static void Restitch ( string RestitcherPackage )
51+ {
52+ // !!!!!!!------------------------------NOTE------------------------------------!!!!!!
53+ // !!!!!!! This code is manually copied into pkg\common\RestitchPackage.targets !!!!!!
54+ // !!!!!!!------------------------------NOTE------------------------------------!!!!!!
55+ //
56+ // vvvvvvvvvvvvvvvvvvvvvvvvvvvvv START HERE vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
57+ try
58+ {
59+ if ( Directory . Exists ( RestitcherPackage ) )
60+ {
61+ using ( var writer = File . CreateText ( "obj/tensorflow_redist_build_log.txt" ) )
62+ {
63+ foreach ( var p in Directory . EnumerateFiles ( RestitcherPackage , "*" , SearchOption . AllDirectories ) )
64+ {
65+
66+ var primaryFile = Path . GetFullPath ( p ) ;
67+ writer . WriteLine ( "Found primary file at {0}" , primaryFile ) ;
68+
69+ // See if there are fragments in the parallel nuget packages. If the primary is
70+ // some-package-primary\runtimes\....\a.so
71+ // some-package-primary\runtimes\....\a.so.sha
72+ // then the expected fragments are
73+ // some-package-fragment1\fragments\....\a.so
74+ // some-package-fragment2\fragments\....\a.so
75+ // some-package-fragment3\fragments\....\a.so
76+ // some-package-fragment4\fragments\....\a.so
77+ // some-package-fragment5\fragments\....\a.so
78+ // some-package-fragment6\fragments\....\a.so
79+ // some-package-fragment7\fragments\....\a.so
80+ // some-package-fragment8\fragments\....\a.so
81+ // some-package-fragment9\fragments\....\a.so
82+ // some-package-fragment10\fragments\....\a.so
83+ var shaFile = primaryFile + ".sha" ;
84+ var fragmentFile1 = primaryFile . Replace ( "-primary" , "-fragment1" ) . Replace ( "runtimes" , "fragments" ) + ".fragment1" ;
85+ var fragmentFile2 = primaryFile . Replace ( "-primary" , "-fragment2" ) . Replace ( "runtimes" , "fragments" ) + ".fragment2" ;
86+ var fragmentFile3 = primaryFile . Replace ( "-primary" , "-fragment3" ) . Replace ( "runtimes" , "fragments" ) + ".fragment3" ;
87+ var fragmentFile4 = primaryFile . Replace ( "-primary" , "-fragment4" ) . Replace ( "runtimes" , "fragments" ) + ".fragment4" ;
88+ var fragmentFile5 = primaryFile . Replace ( "-primary" , "-fragment5" ) . Replace ( "runtimes" , "fragments" ) + ".fragment5" ;
89+
90+
91+ if ( File . Exists ( fragmentFile1 ) ) writer . WriteLine ( "Found fragment file at {0}" , fragmentFile1 ) ;
92+ if ( File . Exists ( fragmentFile2 ) ) writer . WriteLine ( "Found fragment file at {0}" , fragmentFile2 ) ;
93+ if ( File . Exists ( fragmentFile3 ) ) writer . WriteLine ( "Found fragment file at {0}" , fragmentFile3 ) ;
94+ if ( File . Exists ( fragmentFile4 ) ) writer . WriteLine ( "Found fragment file at {0}" , fragmentFile4 ) ;
95+ if ( File . Exists ( fragmentFile5 ) ) writer . WriteLine ( "Found fragment file at {0}" , fragmentFile5 ) ;
96+
97+ if ( File . Exists ( fragmentFile1 ) )
98+ {
99+ var tmpFile = Path . GetTempFileName ( ) ;
100+
101+ {
102+ writer . WriteLine ( "Writing restored primary file at {0}" , tmpFile ) ;
103+ using ( var os = File . OpenWrite ( tmpFile ) )
104+ {
105+
106+ //writer.WriteLine("Writing bytes from {0} to {1}", primaryFile, tmpFile);
107+ //var primaryBytes = File.ReadAllBytes(primaryFile);
108+
109+ //os.Write(primaryBytes, 0, primaryBytes.Length);
110+ if ( File . Exists ( fragmentFile1 ) )
111+ {
112+ writer . WriteLine ( "Writing fragment bytes from {0} to {1}" , fragmentFile1 , tmpFile ) ;
113+ var fragmentBytes1 = File . ReadAllBytes ( fragmentFile1 ) ;
114+ os . Write ( fragmentBytes1 , 0 , fragmentBytes1 . Length ) ;
115+ }
116+ if ( File . Exists ( fragmentFile2 ) )
117+ {
118+ writer . WriteLine ( "Writing fragment bytes from {0} to {1}" , fragmentFile2 , tmpFile ) ;
119+ var fragmentBytes2 = File . ReadAllBytes ( fragmentFile2 ) ;
120+ os . Write ( fragmentBytes2 , 0 , fragmentBytes2 . Length ) ;
121+ }
122+ if ( File . Exists ( fragmentFile3 ) )
123+ {
124+ writer . WriteLine ( "Writing fragment bytes from {0} to {1}" , fragmentFile3 , tmpFile ) ;
125+ var fragmentBytes3 = File . ReadAllBytes ( fragmentFile3 ) ;
126+ os . Write ( fragmentBytes3 , 0 , fragmentBytes3 . Length ) ;
127+ }
128+ if ( File . Exists ( fragmentFile4 ) )
129+ {
130+ writer . WriteLine ( "Writing fragment bytes from {0} to {1}" , fragmentFile4 , tmpFile ) ;
131+ var fragmentBytes4 = File . ReadAllBytes ( fragmentFile4 ) ;
132+ os . Write ( fragmentBytes4 , 0 , fragmentBytes4 . Length ) ;
133+ }
134+ if ( File . Exists ( fragmentFile5 ) )
135+ {
136+ writer . WriteLine ( "Writing fragment bytes from {0} to {1}" , fragmentFile5 , tmpFile ) ;
137+ var fragmentBytes5 = File . ReadAllBytes ( fragmentFile5 ) ;
138+ os . Write ( fragmentBytes5 , 0 , fragmentBytes5 . Length ) ;
139+ }
140+ }
141+ }
142+
143+ var shaExpected = File . Exists ( shaFile ) ? File . ReadAllText ( shaFile ) . ToUpper ( ) : "" ;
144+ writer . WriteLine ( $ "real sha: { shaExpected } ") ;
145+
146+ using ( var sha256Hash = System . Security . Cryptography . SHA256 . Create ( ) )
147+ {
148+ using ( var os2 = File . OpenRead ( tmpFile ) )
149+ {
150+
151+ byte [ ] bytes = sha256Hash . ComputeHash ( os2 ) ;
152+ var builder = new System . Text . StringBuilder ( ) ;
153+ for ( int i = 0 ; i < bytes . Length ; i ++ )
154+ {
155+ builder . Append ( bytes [ i ] . ToString ( "x2" ) ) ;
156+ }
157+ var shaReconstituted = builder . ToString ( ) . ToUpper ( ) ;
158+ if ( shaExpected != shaReconstituted )
159+ {
160+ string msg =
161+ $ "Error downloading and reviving packages. Reconsituted file contents have incorrect SHA\n \t Expected SHA: ${ shaExpected } \n \t Actual SHA: ${ shaReconstituted } \n \t File was reconstituted from:"
162+ + $ "\n \t { primaryFile } (length ${ new FileInfo ( primaryFile ) . Length } )"
163+ + ( File . Exists ( fragmentFile1 ) ? $ "\n \t { fragmentFile1 } (length ${ new FileInfo ( fragmentFile1 ) . Length } )": "" )
164+ + ( File . Exists ( fragmentFile2 ) ? $ "\n \t { fragmentFile2 } (length ${ new FileInfo ( fragmentFile2 ) . Length } )": "" )
165+ + ( File . Exists ( fragmentFile3 ) ? $ "\n \t { fragmentFile3 } (length ${ new FileInfo ( fragmentFile3 ) . Length } )": "" )
166+ + ( File . Exists ( fragmentFile4 ) ? $ "\n \t { fragmentFile4 } (length ${ new FileInfo ( fragmentFile4 ) . Length } )": "" )
167+ + ( File . Exists ( fragmentFile5 ) ? $ "\n \t { fragmentFile5 } (length ${ new FileInfo ( fragmentFile5 ) . Length } )": "" ) ;
168+ writer . WriteLine ( msg ) ;
169+ throw new Exception ( msg ) ;
170+ }
171+ }
172+
173+ }
174+
175+ writer . WriteLine ( "Deleting {0}" , primaryFile ) ;
176+ File . Delete ( primaryFile ) ;
177+ if ( File . Exists ( primaryFile ) )
178+ throw new Exception ( "wtf?" ) ;
179+
180+ writer . WriteLine ( "Moving {0} --> {1}" , tmpFile , primaryFile ) ;
181+ File . Move ( tmpFile , primaryFile ) ;
182+
183+ writer . WriteLine ( "Deleting {0}" , fragmentFile1 ) ;
184+ File . Delete ( fragmentFile1 ) ; // free up space and prevent us doing this again
185+
186+ writer . WriteLine ( "Deleting {0}" , fragmentFile2 ) ;
187+ if ( File . Exists ( fragmentFile2 ) )
188+ File . Delete ( fragmentFile2 ) ; // free up space and prevent us doing this again
189+
190+ writer . WriteLine ( "Deleting {0}" , fragmentFile3 ) ;
191+ if ( File . Exists ( fragmentFile3 ) )
192+ File . Delete ( fragmentFile3 ) ; // free up space and prevent us doing this again
193+
194+ writer . WriteLine ( "Deleting {0}" , fragmentFile4 ) ;
195+ if ( File . Exists ( fragmentFile4 ) )
196+ File . Delete ( fragmentFile4 ) ; // free up space and prevent us doing this again
197+
198+ writer . WriteLine ( "Deleting {0}" , fragmentFile5 ) ;
199+ if ( File . Exists ( fragmentFile5 ) )
200+ File . Delete ( fragmentFile5 ) ; // free up space and prevent us doing this again
201+ }
202+ }
203+ }
204+ }
205+ }
206+ catch ( Exception ex )
207+ {
208+ Console . Error . WriteLine ( ex . ToString ( ) ) ;
209+ Console . Error . WriteLine ( ex . StackTrace ) ;
210+ }
211+ // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ END HERE^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
212+ }
0 commit comments