Skip to content

Commit

Permalink
Merge branch 'master' into creation-time
Browse files Browse the repository at this point in the history
  • Loading branch information
peterwillis authored Feb 9, 2024
2 parents 41d9c40 + 0679355 commit 5f7b028
Show file tree
Hide file tree
Showing 22 changed files with 457 additions and 31 deletions.
20 changes: 18 additions & 2 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,13 @@ on:
- 'pr/**'
schedule:
- cron: 1 1 * * *
workflow_dispatch:
inputs:
debug_enabled:
type: boolean
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
required: false
default: false

jobs:
# Determine the latest go versions
Expand Down Expand Up @@ -42,7 +49,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ needs.go-versions.outputs.latest }}
- name: Install revive
Expand Down Expand Up @@ -76,18 +83,27 @@ jobs:
go_version: ${{ needs.go-versions.outputs.unstable }}
steps:
- uses: actions/checkout@v4
- name: Set cores to get stored as "core"
run: sudo bash -c 'echo "core" > /proc/sys/kernel/core_pattern'
- name: Run tests
run: make test-containers-test "CEPH_VERSION=${{ matrix.ceph_version }}" "GO_VERSION=${{ matrix.go_version }}" "RESULTS_DIR=$PWD/_results"
# Enable tmate debugging of manually-triggered workflows if the input option was provided
- name: Setup tmate session
uses: mxschmitt/action-tmate@v3
if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
- name: Clean up test containers
if: always()
run: make test-containers-clean "CEPH_VERSION=${{ matrix.ceph_version }}"
- name: Archive test results
uses: actions/upload-artifact@v3
if: always()
uses: actions/upload-artifact@v4
with:
name: "go-ceph-results-${{ matrix.ceph_version }}-${{ matrix.go_version }}"
path: |
_results/
retention-days: 30
- name: Check API Versions and Aging
if: always()
run: |
if [ -f _results/implements.json ]; then
./contrib/apiage.py
Expand Down
5 changes: 1 addition & 4 deletions .github/workflows/report-stable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
- name: Run makefile
run: make api-report-issuetemplate RESULTS_DIR=_results
- name: Archive test results
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: "go-ceph-api-updates"
path: |
Expand All @@ -32,6 +32,3 @@ jobs:
with:
filename: _results/issue.md
update_existing: true


# TODO: teach this thing how to file an issue automatically
2 changes: 1 addition & 1 deletion .github/workflows/stale.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
stale:
runs-on: ubuntu-latest
steps:
- uses: actions/stale@v8
- uses: actions/stale@v9
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
stale-issue-message: 'This issue has been automatically marked as stale because it has not had recent activity. It will be closed in 14 days if no further activity occurs. Thank you for your contribution.'
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ go test -tags pacific ....

| go-ceph version | Supported Ceph Versions | Deprecated Ceph Versions |
| --------------- | ------------------------| -------------------------|
| v0.25.0 | pacific, quincy, reef | nautilus, octopus |
| v0.24.0 | pacific, quincy, reef | nautilus, octopus |
| v0.23.0 | pacific, quincy, reef | nautilus, octopus |
| v0.22.0 | pacific, quincy | nautilus, octopus |
Expand Down
3 changes: 2 additions & 1 deletion cephfs/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,14 @@ const (
// ErrNotConnected may be returned when client is not connected
// to a cluster.
ErrNotConnected = cephFSError(-C.ENOTCONN)
// ErrNotExist indicates a non-specific missing resource.
ErrNotExist = cephFSError(-C.ENOENT)
)

// Private errors:

const (
errInvalid = cephFSError(-C.EINVAL)
errNameTooLong = cephFSError(-C.ENAMETOOLONG)
errNoEntry = cephFSError(-C.ENOENT)
errRange = cephFSError(-C.ERANGE)
)
4 changes: 2 additions & 2 deletions cephfs/path_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func TestRemoveDir(t *testing.T) {
assert.NoError(t, err)

_, err = mount.Statx(dirname, StatxBasicStats, 0)
assert.Equal(t, err, errNoEntry)
assert.Equal(t, err, ErrNotExist)
}

func TestLink(t *testing.T) {
Expand Down Expand Up @@ -367,7 +367,7 @@ func TestStatx(t *testing.T) {
st, err = mount.Statx(dirname, StatxBasicStats, 0)
assert.Error(t, err)
assert.Nil(t, st)
assert.Equal(t, errNoEntry, err)
assert.Equal(t, ErrNotExist, err)
})

t.Run("invalidMount", func(t *testing.T) {
Expand Down
30 changes: 28 additions & 2 deletions docs/api-status.json
Original file line number Diff line number Diff line change
Expand Up @@ -1918,7 +1918,14 @@
"became_stable_version": "v0.24.0"
}
],
"preview_api": []
"preview_api": [
{
"name": "Image.Resize2",
"comment": "Resize2 resizes an rbd image and allows configuration of allow_shrink and a callback function. The callback\nfunction will be called with the first argument as the progress, the second argument as the total, and the third\nargument as an opaque value that is passed to the Resize2 function's data argument in each callback execution.\nThe resize operation will be aborted if the progress callback returns a non-zero value.\n\nImplements:\n\n\tint rbd_resize2(rbd_image_t image, uint64_t size, allow_shrink bool, librbd_progress_fn_t cb, void *cbdata);\n",
"added_in_version": "v0.25.0",
"expected_stable_version": "v0.27.0"
}
]
},
"rbd/admin": {
"stable_api": [
Expand Down Expand Up @@ -1993,7 +2000,26 @@
]
},
"rgw/admin": {
"preview_api": [],
"preview_api": [
{
"name": "API.GetInfo",
"comment": "GetInfo - https://docs.ceph.com/en/latest/radosgw/adminops/#info\n",
"added_in_version": "v0.25.0",
"expected_stable_version": "v0.27.0"
},
{
"name": "API.GetBucketQuota",
"comment": "GetBucketQuota - https://docs.ceph.com/en/latest/radosgw/adminops/#get-bucket-quota\n",
"added_in_version": "$NEXT_RELEASE",
"expected_stable_version": "$NEXT_RELEASE_STABLE"
},
{
"name": "API.SetBucketQuota",
"comment": "SetBucketQuota - https://docs.ceph.com/en/latest/radosgw/adminops/#set-bucket-quota\n",
"added_in_version": "$NEXT_RELEASE",
"expected_stable_version": "$NEXT_RELEASE_STABLE"
}
],
"stable_api": [
{
"name": "API.ListBuckets",
Expand Down
14 changes: 13 additions & 1 deletion docs/api-status.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@ No Preview/Deprecated APIs found. All APIs are considered stable.

## Package: rbd

### Preview APIs

Name | Added in Version | Expected Stable Version |
---- | ---------------- | ----------------------- |
Image.Resize2 | v0.25.0 | v0.27.0 |

### Deprecated APIs

Name | Deprecated in Version | Expected Removal Version |
Expand All @@ -30,7 +36,13 @@ No Preview/Deprecated APIs found. All APIs are considered stable.

## Package: rgw/admin

No Preview/Deprecated APIs found. All APIs are considered stable.
### Preview APIs

Name | Added in Version | Expected Stable Version |
---- | ---------------- | ----------------------- |
API.GetInfo | v0.25.0 | v0.27.0 |
API.GetBucketQuota | $NEXT_RELEASE | $NEXT_RELEASE_STABLE |
API.SetBucketQuota | $NEXT_RELEASE | $NEXT_RELEASE_STABLE |

## Package: common/admin/manager

Expand Down
15 changes: 14 additions & 1 deletion entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -257,8 +257,21 @@ test_pkg() {
testargs+=("-memprofile" "${pkg}.mem.out")
fi

show go test -v "${testargs[@]}" "./${pkg}"
ulimit -c unlimited
testbin="./${pkg}/${pkg##*/}.test"
show go test -v "${testargs[@]}" -o "${testbin}" "./${pkg}"
ret=$(($?+ret))
if ls "${pkg}"/core.* >/dev/null 2>&1; then
echo "Found coredump"
sleep 5
coredump="./${pkg}/${pkg##*/}.core"
mv "${pkg}"/core.* "${coredump}"
chmod 644 "${coredump}"
echo "set auto-load safe-path /" >>/root/.gdbinit
gdb "${testbin}" "${coredump}" -ex bt -ex q | cat
mkdir -p "${RESULTS_DIR}"
mv "${testbin}" "${coredump}" "${RESULTS_DIR}/"
fi
grep -v "^mode: count" "${pkg}.cover.out" >> "cover.out"
return ${ret}
}
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ module github.com/ceph/go-ceph
go 1.19

require (
github.com/aws/aws-sdk-go v1.47.1
github.com/aws/aws-sdk-go v1.50.9
github.com/gofrs/uuid v4.4.0+incompatible
github.com/stretchr/testify v1.8.4
golang.org/x/sys v0.13.0
golang.org/x/sys v0.16.0
)

require (
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
github.com/aws/aws-sdk-go v1.47.1 h1:j9ih0Ashcw8tQcnfqNimBM8ARQ/CMpoBwjKue1D6Fuk=
github.com/aws/aws-sdk-go v1.47.1/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go v1.50.9 h1:yX66aKnEtRc/uNV/1EH8CudRT5aLwVwcSwTBphuVPt8=
github.com/aws/aws-sdk-go v1.50.9/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand All @@ -15,8 +15,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
21 changes: 17 additions & 4 deletions micro-osd.sh
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,15 @@ MON_NAME="a"
MGR_NAME="x"
MIRROR_ID="m"
RGW_ID="r"
S3_ACCESS_KEY=2262XNX11FZRR44XWIRD
S3_SECRET_KEY=rmtuS1Uj1bIC08QFYGW18GfSHAbkPqdsuYynNudw
HOST_IP=$(getent ahostsv4 "${HOSTNAME}" | grep STREAM | head -n 1 | awk '{print $1}')

# Following are examples for S3 credentials taken from official AWS docs:
# https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html#access-keys-and-secret-access-keys
# These does not represent real/valid credentials for AWS in any form.
# They are exclusively used for testing S3 compatible API from Ceph RGW.
S3_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE
S3_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY

HOST_IP=$(getent ahostsv4 "${HOSTNAME}" | grep STREAM | head -n 1 | awk '{print $1}')
FSID="$(uuidgen)"
export CEPH_CONF=${DIR}/ceph.conf

Expand Down Expand Up @@ -164,6 +169,10 @@ launch_radosgw() {
radosgw-admin user create --uid admin --display-name "Admin User" --caps "buckets=*;users=*;usage=read;metadata=read" --access-key="$S3_ACCESS_KEY" --secret-key="$S3_SECRET_KEY"
}

launch_radosgw2() {
radosgw-admin caps add --uid=admin --caps="info=read"
}

selftest() {
ceph --version
ceph status
Expand All @@ -183,9 +192,12 @@ if [ -z "$FEATURESET" ] ; then
nautilus|octopus)
FEATURESET="mon osd mgr mds rbd-mirror rgw selftest"
;;
*)
pacific)
FEATURESET="mon osd mgr mds mds2 rbd-mirror cephfs-mirror rgw selftest"
;;
*)
FEATURESET="mon osd mgr mds mds2 rbd-mirror cephfs-mirror rgw rgw2 selftest"
;;
esac
fi

Expand All @@ -200,6 +212,7 @@ for fname in ${FEATURESET} ; do
rbd-mirror) launch_rbd_mirror ;;
cephfs-mirror) launch_cephfs_mirror ;;
rgw|radosgw) launch_radosgw ;;
rgw2|radosgw2) launch_radosgw2 ;;
selftest) selftest ;;
*)
echo "Invalid feature: ${fname}"
Expand Down
77 changes: 77 additions & 0 deletions rbd/resize.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
//go:build ceph_preview

package rbd

/*
#cgo LDFLAGS: -lrbd
#define _POSIX_C_SOURCE 200112L
#undef _GNU_SOURCE
#include <errno.h>
#include <stdlib.h>
#include <rados/librados.h>
#include <rbd/librbd.h>
extern int resize2Callback(uint64_t, uint64_t, uintptr_t);
// inline wrapper to cast uintptr_t to void*
static inline int wrap_rbd_resize2(
rbd_image_t image, uint64_t size, bool allow_shrink, uintptr_t arg) {
return rbd_resize2(
image, size, allow_shrink, (librbd_progress_fn_t)resize2Callback, (void*)arg);
};
*/
import "C"

import (
"github.com/ceph/go-ceph/internal/callbacks"
)

// Resize2ProgressCallback is the callback function type for Image.Resize2.
type Resize2ProgressCallback func(progress uint64, total uint64, data interface{}) int

var resizeCallbacks = callbacks.New()

type resizeProgressCallbackCtx struct {
callback Resize2ProgressCallback
data interface{}
}

//export resize2Callback
func resize2Callback(
offset, total C.uint64_t, index uintptr,
) C.int {
v := resizeCallbacks.Lookup(index)
ctx := v.(resizeProgressCallbackCtx)
return C.int(ctx.callback(uint64(offset), uint64(total), ctx.data))
}

// Resize2 resizes an rbd image and allows configuration of allow_shrink and a callback function. The callback
// function will be called with the first argument as the progress, the second argument as the total, and the third
// argument as an opaque value that is passed to the Resize2 function's data argument in each callback execution.
// The resize operation will be aborted if the progress callback returns a non-zero value.
//
// Implements:
//
// int rbd_resize2(rbd_image_t image, uint64_t size, allow_shrink bool, librbd_progress_fn_t cb, void *cbdata);
func (image *Image) Resize2(size uint64, allowShrink bool, cb Resize2ProgressCallback, data interface{}) error {
// the provided callback must be a real function
if cb == nil {
return rbdError(C.EINVAL)
}

if err := image.validate(imageIsOpen); err != nil {
return err
}

ctx := resizeProgressCallbackCtx{
callback: cb,
data: data,
}
cbIndex := resizeCallbacks.Add(ctx)
defer resizeCallbacks.Remove(cbIndex)

ret := C.wrap_rbd_resize2(image.image, C.uint64_t(size), C.bool(allowShrink), C.uintptr_t(cbIndex))

return getError(ret)

}
Loading

0 comments on commit 5f7b028

Please sign in to comment.