Merge pull request #472 from qilingframework/dev

getting ready for 1.1.1

Author: xwings

.github/workflows/pythonpublish.yml

@@ -16,7 +16,7 @@ jobs:
         pip install setuptools wheel
     - name: Build distribution 📦
       run: |
-        pip install -r requirements.txt
+        pip install .
         python setup.py sdist bdist_wheel
     - uses: actions/upload-artifact@v2
       with:

.gitignore

@@ -14,9 +14,9 @@ build
 dist
 
 # examples 
-examples/**/registry/
-examples/**/dlls/
-examples/**/Windows/
+# examples/**/registry/
+# examples/**/dlls/
+# examples/**/Windows/
 
 # test and logs
 jexamples/

.travis.yml

@@ -9,8 +9,7 @@ matrix:
       python: "3.6"
       cache: pip
       install:
-        - pip3 install -r requirements.txt
-        - python3 setup.py install
+        - pip3 install .
       before_script:
         - cd tests
       script:
@@ -23,9 +22,9 @@ matrix:
       python: "3.7"
       cache: pip
       install:
-        - pip3 install flake8 -r requirements.txt
+        - pip3 install flake8
         - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        - python3 setup.py install
+        - pip3 install .
       before_script:
         - cd tests
       script:
@@ -43,7 +42,6 @@ matrix:
         - pip3 install --upgrade pip
       install:
 #        - pip3 install wheel 'capstone>=4.0.1' 'pefile>=2019.4.18' 'python-registry>=1.3.1' 'unicorn>=1.0.2rc4'
-        - pip3 install -r requirements.txt
 #        - |
 #          if [ ! -f $HOME/dist/keystone*.whl ]; then
 #               git clone https://github.com/keystone-engine/keystone && \
@@ -54,7 +52,7 @@ matrix:
 #              sudo pip3 install $HOME/dist/*.whl;
 #          fi
         - cd $TRAVIS_BUILD_DIR 
-        - python3 setup.py install 
+        - pip3 install . # Workaround since building from keystone sources doesn't work on Macos. Not sure why and I guess it's due to Travis. :/
       before_script:
         - ./examples/scripts/collectdylib.sh
         - cd tests
@@ -85,7 +83,6 @@ matrix:
         #- python -m pip install --upgrade pip
       install:
 #        - pip3 install wheel 'capstone>=4.0.1' 'pefile>=2019.4.18' 'python-registry>=1.3.1' 'unicorn>=1.0.2rc3'
-        - pip3 install -r requirements.txt
 #        - |
 #          if [ ! -f $HOME/dist/keystone*.zip ]; then
 #               git clone https://github.com/keystone-engine/keystone && \
@@ -104,7 +101,7 @@ matrix:
 #              cp $HOME/dist/keystone.dll /c/Python36/Lib/site-packages/keystone/;
 #          fi
         - cd $TRAVIS_BUILD_DIR
-        - python setup.py install
+        - pip3 install .
 #        - cp /c/Program\ Files\ \(x86\)/keystone/lib/keystone.dll /c/Python36/Lib/site-packages/keystone/
       before_script:
         - cmd.exe //C 'examples\scripts\dllscollector.bat'
@@ -135,7 +132,7 @@ matrix:
 #        - docker exec qiling pip3 install -r requirements.txt
 #        - docker exec qiling python3 setup.py install
       script:
-        - docker run -it --rm -v ${TRAVIS_BUILD_DIR}:/qiling qilingframework/qiling:dev bash -c "pip3 install -r requirements.txt && python3 setup.py install && cd tests && ./test_elf.sh"
+        - docker run -it --rm -v ${TRAVIS_BUILD_DIR}:/qiling qilingframework/qiling:dev bash -c "pip3 install . && cd tests && ./test_elf.sh"
 #        - docker exec qiling bash -c "cd tests && ./test_elf.sh"
 
 #    - name: "Python 3.6 on WSL1 Ubuntu"

CREDITS.TXT

@@ -22,6 +22,7 @@ Earl MARCUS (klks84) klks84@gmail.com
 
 Key Contributors (in no particular order)
 =========================================
+lazymio
 kabeor
 0ssigeno
 liba2k

ChangeLog

@@ -1,6 +1,21 @@
 This file details the changelog of Qiling Framework.
 
 
+[Version 1.2-alpha1]: August [someDAY], 2020
+
+- Fix Windows "import resource" issue
+- Add ql.save and ql.restore aka Qiling's Snapshot
+- Add ql.os.fd.save() and ql.os.fd.restore() to save file descriptor
+- Add IDA Plugin
+- Test, patch and make sure arm_thumb can work on its own
+- Added Qiling Debugger - Currently only works with MIPS
+- Add experimental 8086 and DOS support.
+- Fix path transformation on Windows when running Linux.
+- Idaplugin able to instrument code now
+- Refactor ql.fs_mapper (now ql.os.fs_mapper).
+
+
+------------------------------------
 [Version 1.1]: July 24th, 2020
 
 - More refactors and bug fixes

Dockerfile

@@ -11,6 +11,7 @@ RUN apk add --no-cache \
     libtool \
     automake \
     autoconf \
+    libmagic \
     g++ \
     linux-headers \
     git \
@@ -20,8 +21,7 @@ RUN apk add --no-cache \
 
 RUN git clone -b dev https://github.com/qilingframework/qiling.git \
     && cd qiling \
-    && pip3 install -r requirements.txt \
-    && python3 setup.py install \ 
+    && pip3 install . \ 
     && rm -rf /tmp/*
 
 WORKDIR /qiling

README.md

@@ -9,9 +9,9 @@
 
 Qiling is an advanced binary emulation framework, with the following features:
 
-- Cross platform: Windows, MacOS, Linux, BSD, UEFI
-- Cross architecture: X86, X86_64, Arm, Arm64, MIPS
-- Multiple file formats: PE, MachO, ELF
+- Cross platform: Windows, MacOS, Linux, BSD, UEFI, DOS
+- Cross architecture: X86, X86_64, Arm, Arm64, MIPS, 8086
+- Multiple file formats: PE, MachO, ELF, COM
 - Emulate & sandbox machine code in a isolated environment
 - Supports cross architecture and platform debugging capabilities
 - Provide high level API to setup & configure the sandbox
@@ -49,7 +49,7 @@ Qemu usermode does similar thing to our emulator, that is to emulate whole execu
 - Qiling is a true analysis framework, that allows you to build your own dynamic analysis tools on top (in friendly Python language). Meanwhile, Qemu is just a tool, not a framework.
 - Qiling can perform dynamic instrumentation, and can even hotpatch code at runtime. Qemu does not do either.
 - Not only working cross-architecture, Qiling is also cross-platform, so for example you can run Linux ELF file on top of Windows. In contrast, Qemu usermode only run binary of the same OS, such as Linux ELF on Linux, due to the way it forwards syscall from emulated code to native OS.
-- Qiling supports more platforms, including Windows, MacOS, Linux & BSD. Qemu usermode can only handles Linux & BSD.
+- Qiling supports more platforms, including Windows, MacOS, Linux & BSD. Qemu usermode can only handle Linux & BSD.
 
 ---
 
@@ -121,7 +121,7 @@ The below Youtube video shows how the above example works.
 
 #### Fuzzing with Qiling Unicornalf
 
-- More information on fuzzing with Qiling Unicornalf can be found [here](https://github.com/qilingframework/qiling/tree/dev/examples/fuzzing/README.md).
+- More information on fuzzing with Qiling Unicornalf can be found [here](https://github.com/qilingframework/qiling/blob/dev/examples/fuzz_example_x64/README.md).
 
 [![qiling DEMO 2: Fuzzing with Qiling Unicornalf](https://raw.githubusercontent.com/qilingframework/qilingframework.github.io/master/images/qilingfzz-s.png)](https://raw.githubusercontent.com/qilingframework/qiling/dev/examples/fuzzing/qilingfzz.png "Demo #2 Fuzzing with Qiling Unicornalf")
 
@@ -205,9 +205,27 @@ Contact us at email info@qiling.io, or via Twitter [@qiling_io](https://twitter.
 
 #### Key Contributors (in no particular order)
 
+- lazymio
 - kabeor
 - 0ssigeno
 - liba2k
 - assafcarlsbad
 - ucgJhe
 - jhumble
+
+
+#### Donations:
+
+- Opensource project is never an easy task. We value any help from the community
+- Please drop me an email kj@qiling.io after making the donation, i will mail you some swag as a token of appreciation
+
+##### XMR: 
+- 46T1c5taWuP6G4XvAG5shC6a7eai4Qe4HPFj5qEGyJzzMVRa9M9MR4DbNbbSDKtbgNR6bvWyj32Wb3HySYZuDqUp2GCr52o
+##### DASH: 
+- XhTsLXTQEhN5F7hKtq8HV867um3HZuXvF9
+##### ADA: 
+- DdzFFzCqrht8MbmRQL8v86XG5vQHYNC6NQwFkhCW4rsNHMLfzWyxVTce5yFayg6QqJBdL7AapwvFL3fBAoBmPLR9gDbkzLGfVVEGHnNC
+##### ETH: 
+- 0xec095228411d4a232f4d221ad7defcde36eb981f
+##### BTC: 
+- 1NmxDWWak8qtpmYGnXBK1osRNNYH2zxpZs

TODO

@@ -46,14 +46,29 @@ Android
 
 
 -------------------- CHECKLIST before TAG --------------------
-
+Release 
 1. core.py
-    - __version__ = "1.1" + "-rc1"
-2. setup.py
-    - 'Development Status :: 3 - Alpha'
-3. ChangeLog    
+    - __version__ = "1.[x].[x]"
+2. Make sure development status classifer in setup.py is correct.
+    - 'Development Status :: 5 - Production/Stable'
+3. Update ChangeLog    
 4. commit and push
 5. Merge dev into master
-6. git tag 1.1-rc1
-7. git push origin --tags
-8. check for new Pypi package
+6. checkout master
+7. git pull
+6. git tag 1.[x].[x]
+8. git push origin --tags
+9. check for new Pypi package @ https://pypi.org/project/qiling
+
+Pre-release/dev
+1. __version__.py
+    - __version__ = "1.[x].[x]" + "-rc[x]"
+2. Make sure development status classifer in setup.py is correct.
+    - 'Development Status :: 3 - Alpha'
+3. Update ChangeLog
+4. commit and push
+5. checkout dev
+6. git pull
+7. git tag 1.[x].[x]-rc[x]
+8. git push origin --tags
+9. check for new pre-release Pypi package @ https://pypi.org/project/qiling/#history

examples/crackme_x86_linux.py

@@ -50,13 +50,16 @@ def fstat(self):
 def instruction_count(ql, address, size, user_data):
     user_data[0] += 1
 
+def my__llseek(ql, *args, **kw):
+    pass
 
 def run_one_round(payload):
     stdin = MyPipe()
-    ql = Qiling(["rootfs/x86_linux/bin/crackme_linux"], "rootfs/x86_linux", console=False, stdin=stdin,
+    ql = Qiling(["rootfs/x86_linux/bin/crackme_linux"], "rootfs/x86_linux", console=True, stdin=stdin,
                 stdout=sys.stdout, stderr=sys.stderr)
     ins_count = [0]
     ql.hook_code(instruction_count, ins_count)
+    ql.set_syscall("_llseek", my__llseek)
     stdin.write(payload)
     ql.run()
     del stdin

examples/fuzzing/dlink_dir815/.gitignore

@@ -0,0 +1,3 @@
+afl_outputs
+AFLplusplus
+rootfs