Skip to content

Enable Appveyor CI on Windows #29

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 9, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions .appveyor.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# CI on Windows via appveyor
environment:

matrix:
## MINGW
#
- PYTHON: "C:\\Python27"
PYTHON_VERSION: "2.7"
- PYTHON: "C:\\Python34-x64"
PYTHON_VERSION: "3.4"
- PYTHON: "C:\\Python35-x64"
PYTHON_VERSION: "3.5"
- PYTHON: "C:\\Miniconda35-x64"
PYTHON_VERSION: "3.5"
IS_CONDA: "yes"

install:
- set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH%

## Print configuration for debugging.
#
- |
echo %PATH%
uname -a
where python pip pip2 pip3 pip34
python --version
python -c "import struct; print(struct.calcsize('P') * 8)"

- IF "%IS_CONDA%"=="yes" (
conda info -a &
conda install --yes --quiet pip
)
- pip install nose wheel coveralls

## For commits performed with the default user.
- |
git config --global user.email "[email protected]"
git config --global user.name "Travis Runner"

- pip install -e .

build: false

test_script:
- IF "%PYTHON_VERSION%"=="3.5" (
nosetests -v --with-coverage
) ELSE (
nosetests -v
)
15 changes: 8 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ Although memory maps have many advantages, they represent a very limited system
## Overview

[![Build Status](https://travis-ci.org/gitpython-developers/smmap.svg?branch=master)](https://travis-ci.org/gitpython-developers/smmap)
[![Build status](https://ci.appveyor.com/api/projects/status/h8rl7thsr42oc0pf?svg=true&passingText=windows%20OK&failingText=windows%20failed)](https://ci.appveyor.com/project/Byron/smmap)
[![Coverage Status](https://coveralls.io/repos/gitpython-developers/smmap/badge.png)](https://coveralls.io/r/gitpython-developers/smmap)
[![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap)
[![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap)
Expand Down Expand Up @@ -44,15 +45,15 @@ The package was tested on all of the previously mentioned configurations.
[![Documentation Status](https://readthedocs.org/projects/smmap/badge/?version=latest)](https://readthedocs.org/projects/smmap/?badge=latest)

Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program:

```bash
$ pip install smmap
```

As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes.

If you have downloaded the source archive, the package can be installed by running the `setup.py` script:

```bash
$ python setup.py install
```
Expand All @@ -68,17 +69,17 @@ The project is home on github at https://github.com/gitpython-developers/smmap .
The latest source can be cloned from github as well:

* git://github.com/gitpython-developers/smmap.git


For support, please use the git-python mailing list:

* http://groups.google.com/group/git-python


Issues can be filed on github:

* https://github.com/gitpython-developers/smmap/issues


## License Information

Expand Down
12 changes: 9 additions & 3 deletions smmap/buf.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0):
def __del__(self):
self.end_access()

def __enter__(self):
return self

def __exit__(self, exc_type, exc_value, traceback):
self.end_access()

def __len__(self):
return self._size

Expand Down Expand Up @@ -83,15 +89,15 @@ def __getslice__(self, i, j):
# in the previous iteration of this code
pyvers = sys.version_info[:2]
if (3, 0) <= pyvers <= (3, 3):
# Memory view cannot be joined below python 3.4 ...
# Memory view cannot be joined below python 3.4 ...
out = bytes()
while l:
c.use_region(ofs, l)
assert c.is_valid()
d = c.buffer()[:l]
ofs += len(d)
l -= len(d)
# This is slower than the join ... but what can we do ...
# This is slower than the join ... but what can we do ...
out += d
del(d)
# END while there are bytes to read
Expand All @@ -104,7 +110,7 @@ def __getslice__(self, i, j):
d = c.buffer()[:l]
ofs += len(d)
l -= len(d)
# Make sure we don't keep references, as c.use_region() might attempt to free resources, but
# Make sure we don't keep references, as c.use_region() might attempt to free resources, but
# can't unless we use pure bytes
if hasattr(d, 'tobytes'):
d = d.tobytes()
Expand Down
6 changes: 6 additions & 0 deletions smmap/mman.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ def __init__(self, manager=None, regions=None):
def __del__(self):
self._destroy()

def __enter__(self):
return self

def __exit__(self, exc_type, exc_value, traceback):
self._destroy()

def _destroy(self):
"""Destruction code to decrement counters"""
self.unuse_region()
Expand Down
13 changes: 9 additions & 4 deletions smmap/test/lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,9 @@ def __init__(self, size, prefix=''):
self._path = tempfile.mktemp(prefix=prefix)
self._size = size

fp = open(self._path, "wb")
fp.seek(size - 1)
fp.write(b'1')
fp.close()
with open(self._path, "wb") as fp:
fp.seek(size - 1)
fp.write(b'1')

assert os.path.getsize(self.path) == size

Expand All @@ -35,6 +34,12 @@ def __del__(self):
pass
# END exception handling

def __enter__(self):
return self

def __exit__(self, exc_type, exc_value, traceback):
self.__del__()

@property
def path(self):
return self._path
Expand Down
202 changes: 101 additions & 101 deletions smmap/test/test_buf.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,104 +25,104 @@
class TestBuf(TestBase):

def test_basics(self):
fc = FileCreator(self.k_window_test_size, "buffer_test")

# invalid paths fail upon construction
c = man_optimal.make_cursor(fc.path)
self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor
self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large

buf = SlidingWindowMapBuffer() # can create uninitailized buffers
assert buf.cursor() is None

# can call end access any time
buf.end_access()
buf.end_access()
assert len(buf) == 0

# begin access can revive it, if the offset is suitable
offset = 100
assert buf.begin_access(c, fc.size) == False
assert buf.begin_access(c, offset) == True
assert len(buf) == fc.size - offset
assert buf.cursor().is_valid()

# empty begin access keeps it valid on the same path, but alters the offset
assert buf.begin_access() == True
assert len(buf) == fc.size
assert buf.cursor().is_valid()

# simple access
with open(fc.path, 'rb') as fp:
data = fp.read()
assert data[offset] == buf[0]
assert data[offset:offset * 2] == buf[0:offset]

# negative indices, partial slices
assert buf[-1] == buf[len(buf) - 1]
assert buf[-10:] == buf[len(buf) - 10:len(buf)]

# end access makes its cursor invalid
buf.end_access()
assert not buf.cursor().is_valid()
assert buf.cursor().is_associated() # but it remains associated

# an empty begin access fixes it up again
assert buf.begin_access() == True and buf.cursor().is_valid()
del(buf) # ends access automatically
del(c)

assert man_optimal.num_file_handles() == 1

# PERFORMANCE
# blast away with random access and a full mapping - we don't want to
# exaggerate the manager's overhead, but measure the buffer overhead
# We do it once with an optimal setting, and with a worse manager which
# will produce small mappings only !
max_num_accesses = 100
fd = os.open(fc.path, os.O_RDONLY)
for item in (fc.path, fd):
for manager, man_id in ((man_optimal, 'optimal'),
(man_worst_case, 'worst case'),
(static_man, 'static optimal')):
buf = SlidingWindowMapBuffer(manager.make_cursor(item))
assert manager.num_file_handles() == 1
for access_mode in range(2): # single, multi
num_accesses_left = max_num_accesses
num_bytes = 0
fsize = fc.size

st = time()
buf.begin_access()
while num_accesses_left:
num_accesses_left -= 1
if access_mode: # multi
ofs_start = randint(0, fsize)
ofs_end = randint(ofs_start, fsize)
d = buf[ofs_start:ofs_end]
assert len(d) == ofs_end - ofs_start
assert d == data[ofs_start:ofs_end]
num_bytes += len(d)
del d
else:
pos = randint(0, fsize)
assert buf[pos] == data[pos]
num_bytes += 1
# END handle mode
# END handle num accesses

buf.end_access()
assert manager.num_file_handles()
assert manager.collect()
assert manager.num_file_handles() == 0
elapsed = max(time() - st, 0.001) # prevent zero division errors on windows
mb = float(1000 * 1000)
mode_str = (access_mode and "slice") or "single byte"
print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)"
% (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed),
file=sys.stderr)
# END handle access mode
del buf
# END for each manager
# END for each input
os.close(fd)
with FileCreator(self.k_window_test_size, "buffer_test") as fc:

# invalid paths fail upon construction
c = man_optimal.make_cursor(fc.path)
self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor
self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large

buf = SlidingWindowMapBuffer() # can create uninitailized buffers
assert buf.cursor() is None

# can call end access any time
buf.end_access()
buf.end_access()
assert len(buf) == 0

# begin access can revive it, if the offset is suitable
offset = 100
assert buf.begin_access(c, fc.size) == False
assert buf.begin_access(c, offset) == True
assert len(buf) == fc.size - offset
assert buf.cursor().is_valid()

# empty begin access keeps it valid on the same path, but alters the offset
assert buf.begin_access() == True
assert len(buf) == fc.size
assert buf.cursor().is_valid()

# simple access
with open(fc.path, 'rb') as fp:
data = fp.read()
assert data[offset] == buf[0]
assert data[offset:offset * 2] == buf[0:offset]

# negative indices, partial slices
assert buf[-1] == buf[len(buf) - 1]
assert buf[-10:] == buf[len(buf) - 10:len(buf)]

# end access makes its cursor invalid
buf.end_access()
assert not buf.cursor().is_valid()
assert buf.cursor().is_associated() # but it remains associated

# an empty begin access fixes it up again
assert buf.begin_access() == True and buf.cursor().is_valid()
del(buf) # ends access automatically
del(c)

assert man_optimal.num_file_handles() == 1

# PERFORMANCE
# blast away with random access and a full mapping - we don't want to
# exaggerate the manager's overhead, but measure the buffer overhead
# We do it once with an optimal setting, and with a worse manager which
# will produce small mappings only !
max_num_accesses = 100
fd = os.open(fc.path, os.O_RDONLY)
for item in (fc.path, fd):
for manager, man_id in ((man_optimal, 'optimal'),
(man_worst_case, 'worst case'),
(static_man, 'static optimal')):
buf = SlidingWindowMapBuffer(manager.make_cursor(item))
assert manager.num_file_handles() == 1
for access_mode in range(2): # single, multi
num_accesses_left = max_num_accesses
num_bytes = 0
fsize = fc.size

st = time()
buf.begin_access()
while num_accesses_left:
num_accesses_left -= 1
if access_mode: # multi
ofs_start = randint(0, fsize)
ofs_end = randint(ofs_start, fsize)
d = buf[ofs_start:ofs_end]
assert len(d) == ofs_end - ofs_start
assert d == data[ofs_start:ofs_end]
num_bytes += len(d)
del d
else:
pos = randint(0, fsize)
assert buf[pos] == data[pos]
num_bytes += 1
# END handle mode
# END handle num accesses

buf.end_access()
assert manager.num_file_handles()
assert manager.collect()
assert manager.num_file_handles() == 0
elapsed = max(time() - st, 0.001) # prevent zero division errors on windows
mb = float(1000 * 1000)
mode_str = (access_mode and "slice") or "single byte"
print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)"
% (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed),
file=sys.stderr)
# END handle access mode
del buf
# END for each manager
# END for each input
os.close(fd)
Loading