forked from google/myanmar-tools
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
9697316
commit 91c2b57
Showing
26 changed files
with
2,228 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| # Miscellaneous | ||
| *.class | ||
| *.log | ||
| *.pyc | ||
| *.swp | ||
| .DS_Store | ||
| .atom/ | ||
| .buildlog/ | ||
| .history | ||
| .svn/ | ||
|
|
||
| # IntelliJ related | ||
| *.iml | ||
| *.ipr | ||
| *.iws | ||
| .idea/ | ||
|
|
||
| # The .vscode folder contains launch configuration and tasks you configure in | ||
| # VS Code which you may wish to be included in version control, so this line | ||
| # is commented out by default. | ||
| #.vscode/ | ||
|
|
||
| # Flutter/Dart/Pub related | ||
| **/doc/api/ | ||
| .dart_tool/ | ||
| .flutter-plugins | ||
| .flutter-plugins-dependencies | ||
| .packages | ||
| .pub-cache/ | ||
| .pub/ | ||
| build/ | ||
|
|
||
| # Android related | ||
| **/android/**/gradle-wrapper.jar | ||
| **/android/.gradle | ||
| **/android/captures/ | ||
| **/android/gradlew | ||
| **/android/gradlew.bat | ||
| **/android/local.properties | ||
| **/android/**/GeneratedPluginRegistrant.java | ||
|
|
||
| # iOS/XCode related | ||
| **/ios/**/*.mode1v3 | ||
| **/ios/**/*.mode2v3 | ||
| **/ios/**/*.moved-aside | ||
| **/ios/**/*.pbxuser | ||
| **/ios/**/*.perspectivev3 | ||
| **/ios/**/*sync/ | ||
| **/ios/**/.sconsign.dblite | ||
| **/ios/**/.tags* | ||
| **/ios/**/.vagrant/ | ||
| **/ios/**/DerivedData/ | ||
| **/ios/**/Icon? | ||
| **/ios/**/Pods/ | ||
| **/ios/**/.symlinks/ | ||
| **/ios/**/profile | ||
| **/ios/**/xcuserdata | ||
| **/ios/.generated/ | ||
| **/ios/Flutter/App.framework | ||
| **/ios/Flutter/Flutter.framework | ||
| **/ios/Flutter/Flutter.podspec | ||
| **/ios/Flutter/Generated.xcconfig | ||
| **/ios/Flutter/ephemeral | ||
| **/ios/Flutter/app.flx | ||
| **/ios/Flutter/app.zip | ||
| **/ios/Flutter/flutter_assets/ | ||
| **/ios/Flutter/flutter_export_environment.sh | ||
| **/ios/ServiceDefinitions.json | ||
| **/ios/Runner/GeneratedPluginRegistrant.* | ||
|
|
||
| # Exceptions to above rules. | ||
| !**/ios/**/default.mode1v3 | ||
| !**/ios/**/default.mode2v3 | ||
| !**/ios/**/default.pbxuser | ||
| !**/ios/**/default.perspectivev3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| Copyright 2021 Sei Thu Htun | ||
|
|
||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
|
|
||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
|
|
||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| # Myanmar Tools Dart(Flutter) Documentation | ||
|
|
||
| This documentation is for Dart(Flutter) specific usage of *Myanmar Tools*. | ||
| See https://github.com/google/myanmar-tools/ for more. | ||
|
|
||
| ## Usage | ||
|
|
||
| To detect Zawgyi, create a final instance of ZawgyiDetector, and call `getZawgyiProbability` with your string. | ||
|
|
||
| ```dart | ||
| final _detector = await ZawGyiDetector.create(); | ||
| double score = _detector.getZawGyiProbability("မ္း"); | ||
| // score is now 0.999772 (very likely Zawgyi) | ||
| ``` | ||
|
|
||
| To convert between Zawgyi and Unicode, use the classe `ZawGyiConverter` as shown below. | ||
|
|
||
| ```dart | ||
| final converter= ZawGyiConverter.create(); | ||
| // Zawgyi to Unicode | ||
| String uniOutput = converter.zawGyiToUnicode('မ္း'); | ||
| // uniOutput is now 'မ်း' | ||
| // Unicode to Zawgyi | ||
| String zawOutput = converter.zawGyiToUnicode('မ်း'); | ||
| // zawOutput is now 'မ္း' | ||
| ``` | ||
|
|
||
| ## Contributing | ||
|
|
||
| Bug reports and pull requests are welcome on GitHub at https://github.com/google/myanmar-tools . This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct. | ||
|
|
||
| ## License | ||
|
|
||
| The package is available as open source under the terms of the [Apache-2.0 License](http://www.apache.org/licenses/LICENSE-2.0). |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| include: package:flutter_lints/flutter.yaml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| library myanmar_tools; | ||
|
|
||
| export 'src/zawgyi_detector.dart'; | ||
| export 'src/zawgyi_converter.dart'; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| import 'dart:typed_data'; | ||
|
|
||
|
|
||
| /// A class that behaves as if it were two Markov chains, called Chain A and Chain B. Whereas a | ||
| /// normal Markov chain would expose the log probability of a transition, this class exposes the | ||
| /// difference between the log probabilities of the two Markov chains. When training, you can specify | ||
| /// which Markov chain a transition should be added to. | ||
| /// | ||
| /// The reasoning behind this class is that it has a smaller data footprint than two separate | ||
| /// Markov chain objects. | ||
| /// | ||
| class BinaryMarkov { | ||
|
|
||
|
|
||
| // Dart Web JavaScript doesn't have the concept of 64-bit integers, so split the tag into two 32-bit ints. | ||
| static const _binaryTagLead = 0x424D4152; | ||
| static const _binaryTagTrail =0x4B4F5620; | ||
|
|
||
| // Current serial format version number, used in association with the magic number. | ||
| static const _binaryVersion = 0; | ||
|
|
||
| late List<List<double>> logProbabilityDifferences=[]; | ||
|
|
||
|
|
||
| /// Creates an instance from a binary data stream. | ||
| /// | ||
| /// @throws Exception If there was a problem reading the data. | ||
| BinaryMarkov(ByteData stream,int offset){ | ||
|
|
||
| final binaryTagLead = stream.getUint32(offset); | ||
| offset += 4; | ||
|
|
||
| if (binaryTagLead != _binaryTagLead) { | ||
| throw Exception('Unexpected magic number lead; expected 0x${_binaryTagLead.toRadixString(16)} but got 0x${binaryTagLead.toRadixString(16)}'); | ||
| } | ||
| final binaryTagTrail = stream.getUint32(offset); | ||
| offset += 4; | ||
| if (binaryTagTrail != _binaryTagTrail) { | ||
| throw Exception('Unexpected magic number trail; expected 0x${_binaryTagTrail.toRadixString(16)} but got0x${binaryTagTrail.toRadixString(16)}'); | ||
| } | ||
|
|
||
| final binaryVersion= stream.getUint32(offset); | ||
| offset+=4; | ||
|
|
||
| if (binaryVersion != _binaryVersion) { | ||
| throw Exception('Unexpected serial version number; expected ${_binaryVersion.toString()} but got ${binaryVersion.toString()}'); | ||
| } | ||
| int size = stream.getInt16(offset); | ||
| offset += 2; | ||
| var logProbabilityDifferences = List.generate(size, (_) =>List.filled(size, 0.0) ); | ||
| for (int i1 = 0; i1 < size; i1++) { | ||
| int entries = stream.getInt16(offset); | ||
| offset += 2; | ||
| double fallback=0; | ||
| if(entries!=0){ | ||
| fallback = stream.getFloat32(offset); | ||
| offset += 4; | ||
| } | ||
| int next = -1; | ||
| for (int i2 = 0; i2 < size; i2++) { | ||
| if (entries > 0 && next < i2) { | ||
| next = stream.getInt16(offset); | ||
| offset += 2; | ||
| entries--; | ||
| } | ||
| if (next == i2) { | ||
| logProbabilityDifferences[i1][i2] = stream.getFloat32(offset); | ||
| offset += 4; | ||
| } else { | ||
| logProbabilityDifferences[i1][i2] = fallback; | ||
| } | ||
| } | ||
| } | ||
| this.logProbabilityDifferences = logProbabilityDifferences; | ||
| } | ||
|
|
||
|
|
||
| /// Gets the difference in log probabilities between chain A and chain B. This behaves as if you | ||
| /// had two Markov chains and called <code> | ||
| /// chainA.getLogProbability(i1, i2) - chainB.getLogProbability(i1, i2)</code>. | ||
| /// | ||
| /// @param i1 The index of the source node to transition from. | ||
| /// @param i2 The index of the destination node to transition to. | ||
| /// @return The difference between A and B in log probability of transitioning from i1 to i2. | ||
| double getLogProbabilityDifference(int i1, int i2) { | ||
| return logProbabilityDifferences[i1][i2]; | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.