Permettre de téléverser plusieurs fichiers à la fois dans admin-upload #182

Merged
vlbeaudoin merged 12 commits from feature/upload-plusieurs-documents into main 2023-12-19 19:50:50 -05:00
9 changed files with 403 additions and 14 deletions

View file

@ -36,6 +36,80 @@ func New(client *http.Client, host, key string, port int, protocol string) (*API
return &API{Voki: voki.New(client, host, key, port, protocol)}, nil
}
func (a *API) UploadDocuments(bucketName string, fileHeaders ...*multipart.FileHeader) (response apiresponse.V1DocumentsPOST, err error) {
if count := len(fileHeaders); count == 0 {
err = fmt.Errorf("api.(*API).UploadDocuments requiert au moins 1 fichier")
return
}
endpoint := fmt.Sprintf("%s://%s:%d/v1/bucket/%s/many",
a.Voki.Protocol,
a.Voki.Host,
a.Voki.Port,
bucketName,
)
// Create new multipart writer
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
// Add files to the request
for i, fileHeader := range fileHeaders {
if fileHeader == nil {
return response, fmt.Errorf("Fichier %d pointe vers une addresse mémoire nulle", i)
}
file, err := fileHeader.Open()
if err != nil {
return response, fmt.Errorf("Impossible de lire le contenu du fichier %d '%s': %s", i, fileHeader.Filename, err)
}
defer file.Close()
fileName, err := url.QueryUnescape(fileHeader.Filename)
if err != nil {
return response, fmt.Errorf("Fichier %d '%s' a un nom invalide et impossible à convertir: %s", i, fileHeader.Filename, err)
}
part, err := writer.CreatePart(fileHeader.Header)
if err != nil {
return response, fmt.Errorf("Impossible d'ajouter %d '%s' au formulaire de téléversement: %s", i, fileName, err)
}
_, err = io.Copy(part, file)
if err != nil {
return response, fmt.Errorf("Impossible d'ajouter le contenu de %d '%s' au formulaire de téléversement: %s", i, fileName, err)
}
}
if err := writer.Close(); err != nil {
return response, fmt.Errorf("Impossible de fermer le io.Writer: %s", err)
}
req, err := http.NewRequest(http.MethodPost, endpoint, body)
if err != nil {
return response, fmt.Errorf("Impossible de produire une requête: %s", err)
}
if err := req.ParseForm(); err != nil {
return response, fmt.Errorf("Impossible de parse le formulaire: %s", err)
}
req.Header.Set("Content-Type", writer.FormDataContentType())
if a.Voki.Key != "" {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", a.Voki.Key))
}
// Send the HTTP request
httpResponse, err := a.Voki.Client.Do(req)
if err != nil {
return response, fmt.Errorf("Impossible d'exécuter la requête http: %s", err)
}
defer httpResponse.Body.Close()
return response, json.NewDecoder(httpResponse.Body).Decode(&response)
}
func (a *API) UploadDocument(bucket string, file_header *multipart.FileHeader) (apiresponse.V1DocumentPOST, error) {
var response apiresponse.V1DocumentPOST
endpoint := fmt.Sprintf("%s://%s:%d",

View file

@ -2,6 +2,7 @@ package apihandler
import (
"context"
"fmt"
"net/http"
"git.agecem.com/agecem/agecem-org/apirequest"
@ -10,6 +11,78 @@ import (
"github.com/minio/minio-go/v7"
)
/*
V1DocumentsPOST permet d'ajouter un object dans un bucket, par multipart/form-data
Example:
Téléverser plusieurs fichiers à cette route avec `curl`:
curl <endpoint> -F 'documents=@example.pdf' -F 'documents=@example.md;type=text/markdown'
*/
func (h *V1Handler) V1DocumentsPOST(c echo.Context) (err error) {
var request apirequest.V1DocumentsPOST
var response apiresponse.V1DocumentsPOST
request.Params.Bucket = c.Param("bucket")
var allowed bool
for allowedBucket := range h.Config.Server.Documents.Buckets {
if request.Params.Bucket == allowedBucket {
allowed = true
}
}
if !allowed {
return c.JSON(apiresponse.NotFoundResponse())
}
form, err := c.MultipartForm()
if err != nil {
response.Message = fmt.Sprintf("Téléversement invalide: %s", err)
response.StatusCode = http.StatusBadRequest
return c.JSON(response.StatusCode, response)
}
if form == nil {
response.Message = "MultipartForm pointe vers une addresse mémoire nil"
response.StatusCode = http.StatusBadRequest
return c.JSON(response.StatusCode, response)
}
if len(form.File) == 0 {
response.Message = "Veuillez sélectionner au moins 1 document à téléverser"
response.StatusCode = http.StatusBadRequest
return c.JSON(response.StatusCode, response)
}
for inputName, inputFileHeaders := range form.File {
if inputName == "documents" {
request.Data.Documents = inputFileHeaders
}
}
if request.Data.Documents == nil {
response.Message = "Impossible d'obtenir les documents depuis le formulaire"
response.StatusCode = http.StatusBadRequest
return c.JSON(response.StatusCode, response)
}
if !request.Complete() {
response.Message = "Requête V1DocumentsPOST incomplète reçue"
response.StatusCode = http.StatusBadRequest
return c.JSON(response.StatusCode, response)
}
response.StatusCode, response.Message = h.MediaClient.UploadFormFiles(request.Params.Bucket, request.Data.Documents)
return c.JSON(response.StatusCode, response)
}
// V1DocumentPOST permet d'ajouter un object dans un bucket, par multipart/form-data
func (h *V1Handler) V1DocumentPOST(c echo.Context) (err error) {
var request apirequest.V1DocumentPOST

View file

@ -12,8 +12,72 @@ import (
"git.agecem.com/agecem/agecem-org/apiresponse"
)
var _ request.Requester[apiresponse.V1DocumentsPOST] = V1DocumentsPOST{}
type V1DocumentsPOST struct {
Data struct {
Documents []*multipart.FileHeader `json:"documents"`
}
Params struct {
Bucket string `json:"bucket"`
}
}
func NewV1DocumentsPOST(bucket string, documents ...*multipart.FileHeader) (request V1DocumentsPOST, err error) {
if bucket == "" {
err = fmt.Errorf("NewV1DocumentsPOST requires non-nil bucket name")
return
}
request.Params.Bucket = bucket
if documents == nil {
err = fmt.Errorf("NewV1DocumentsPOST requires non-nil documents")
return
}
for _, document := range documents {
if document == nil {
err = fmt.Errorf("NewV1DocumentsPOST requires non-nil documents")
return
}
}
request.Data.Documents = documents
return
}
func (request V1DocumentsPOST) Complete() bool {
if request.Data.Documents == nil {
return false
}
for _, document := range request.Data.Documents {
if document == nil {
return false
}
}
return request.Params.Bucket != ""
}
func (request V1DocumentsPOST) Request(v *voki.Voki) (response apiresponse.V1DocumentsPOST, err error) {
if !request.Complete() {
err = fmt.Errorf("Incomplete V1DocumentsPOST request")
return
}
var buf bytes.Buffer
if err = json.NewEncoder(&buf).Encode(request.Data); err != nil {
return
}
return response, v.UnmarshalIfComplete(http.MethodPost, fmt.Sprintf("/v1/bucket/%s/many", request.Params.Bucket), &buf, true, &response)
}
var _ request.Requester[apiresponse.V1DocumentPOST] = V1DocumentPOST{}
// Deprecated: Use V1DocumentsPOST instead
type V1DocumentPOST struct {
Data struct {
Document *multipart.FileHeader `document`
@ -23,6 +87,7 @@ type V1DocumentPOST struct {
}
}
// Deprecated: Use NewV1DocumentsPOST instead
func NewV1DocumentPOST(bucket string, document *multipart.FileHeader) (request V1DocumentPOST, err error) {
if bucket == "" {
err = fmt.Errorf("NewV1DocumentPOST requires non-nil bucket name")

View file

@ -1,11 +1,24 @@
package apiresponse
type DataDocument struct {
Key string
Size int64
}
type V1DocumentsPOST struct {
Response
Data struct {
Bucket string
Documents []DataDocument
}
}
// Deprecated: Use V1DocumentsPOST instead
type V1DocumentPOST struct {
Response
Data struct {
Bucket string
Key string
Size int64
DataDocument
}
}

View file

@ -242,6 +242,12 @@ func RunServer() {
log.Fatal(err)
}
if err := pave.EchoRegister[
apirequest.V1DocumentsPOST,
apiresponse.V1DocumentsPOST](groupV1, &p, "/v1", http.MethodPost, "/bucket/:bucket/many", "Upload documents to specified bucket", "V1DocumentsPOST", v1Handler.V1DocumentsPOST); err != nil {
log.Fatal(err)
}
if err := pave.EchoRegister[
apirequest.V1DocumentPOST,
apiresponse.V1DocumentPOST](groupV1, &p, "/v1", http.MethodPost, "/bucket/:bucket", "Upload document to specified bucket", "V1DocumentPOST", v1Handler.V1DocumentPOST); err != nil {

View file

@ -3,6 +3,10 @@ package media
import (
"context"
"errors"
"fmt"
"mime"
"mime/multipart"
"net/http"
"git.agecem.com/agecem/agecem-org/config"
"github.com/minio/minio-go/v7"
@ -76,3 +80,106 @@ func (m *MediaClient) Seed() ([]string, error) {
return new_buckets, nil
}
func (m *MediaClient) UploadFormFiles(bucketName string, fileHeaders []*multipart.FileHeader) (statusCode int, result string) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ok, err := m.MinioClient.BucketExists(ctx, bucketName)
if err != nil {
return http.StatusInternalServerError, fmt.Sprintf("Erreur lors de vérification d'existence de bucket '%s': %s", bucketName, err)
}
if !ok {
return http.StatusBadRequest, fmt.Sprintf("Bucket '%s' n'existe pas", bucketName)
}
switch count := len(fileHeaders); count {
case 0:
return http.StatusBadRequest, "Veuillez sélectionner au moins 1 document à téléverser"
case 1:
result = "Téléversement de 1 fichier\n"
default:
result = fmt.Sprintf("Téléversement de %d fichiers\n", count)
}
var allowedMediaTypes = []string{"application/pdf", "text/markdown", "text/plain"}
var fileNames []string
for _, fileHeader := range fileHeaders {
fileNames = append(fileNames, fileHeader.Filename)
}
var validFileHeaders []*multipart.FileHeader
for i, fileHeader := range fileHeaders {
// Check for conflicting file names in upload
for j, fileName := range fileNames {
if fileName == fileHeader.Filename && i != j {
return http.StatusBadRequest, fmt.Sprintf("Doublon de nom de fichier '%s' trouvé, les noms de fichiers doivent être uniques", fileName)
}
}
// Check media type
mediaType, _, err := mime.ParseMediaType(fileHeader.Header.Get("Content-Type"))
if err != nil {
return http.StatusBadRequest, fmt.Sprintf("Impossible de déterminer le type de fichier pour %d '%s'.\nPlus de détails: %s", i, fileHeader.Filename, err.Error())
}
var isAllowedMediaType bool
for _, allowedMediaType := range allowedMediaTypes {
if allowedMediaType == mediaType {
isAllowedMediaType = true
}
}
if !isAllowedMediaType {
return http.StatusUnsupportedMediaType, fmt.Sprintf("Type de fichier interdit '%s' pour '%s'.\nTypes de fichiers permis: %s", mediaType, fileHeader.Filename, allowedMediaTypes)
}
// Check for conflicting fileNames with existing files
objectInfo, err := m.MinioClient.StatObject(ctx, bucketName, fileHeader.Filename, minio.StatObjectOptions{})
if err == nil && objectInfo.Key == fileHeader.Filename {
return http.StatusConflict, fmt.Sprintf("Un document au nom '%s' de catégorie '%s' existe déjà et ne peut pas être inséré de cette façon.", fileHeader.Filename, bucketName)
}
switch msg := err.Error(); msg {
case "The specified key does not exist.":
default:
return http.StatusInternalServerError, fmt.Sprintf("Erreur inattendue lors de vérification de conflit de nom de fichier avec la base de données: %s", err)
}
validFileHeaders = append(validFileHeaders, fileHeader)
}
if len(validFileHeaders) == 0 {
return http.StatusOK, "Aucun fichier valide envoyé au serveur, rien à faire."
}
for i, fileHeader := range validFileHeaders {
mediaType, _, err := mime.ParseMediaType(fileHeader.Header.Get("Content-Type"))
if err != nil {
return http.StatusBadRequest, fmt.Sprintf("Impossible de déterminer le type de fichier pour %d '%s'.\nPlus de détails: %s", i, fileHeader.Filename, err.Error())
}
// Get file content
fileContent, err := fileHeader.Open()
if err != nil {
return http.StatusBadRequest, fmt.Sprintf("Impossible de lire le contenu de '%s': %s", fileHeader.Filename, err)
}
defer fileContent.Close()
// Upload file
info, err := m.MinioClient.PutObject(ctx, bucketName, fileHeader.Filename, fileContent, fileHeader.Size, minio.PutObjectOptions{
ContentType: mediaType,
})
if err != nil {
return http.StatusInternalServerError, fmt.Sprintf("Impossible d'ajouter '%s' à la base de donnée: %s", fileHeader.Filename, err)
}
result = fmt.Sprintf("%sDocument %d '%s' de type '%s' et de taille '%d' téléversé à '%s' avec succès\n",
result, i, info.Key, mediaType, info.Size, info.Bucket)
}
return http.StatusCreated, result
}

View file

@ -15,7 +15,12 @@
<div class="formContent">
<div class="formDocUploadDiv">
<p class="formLabel">Document à téléverser</p>
<input class="formDocUpload" type="file" name="document">
<input class="formDocUpload"
type="file"
name="documents"
accept="application/pdf,.md,text/markdown;charset=UTF-8,text/plain"
multiple
/>
</div>
<div class="formSelectDiv">
<label class="formLabel" for="bucket">Type de document</label>

View file

@ -12,7 +12,7 @@
<div class="wrapper adminWrapper">
<h1 class="heading1">Admin</h1>
<div class="adminOptions">
<button class="adminOption" onclick="location.href = '/admin/documents/upload'">Ajouter un document</a>
<button class="adminOption" onclick="location.href = '/admin/documents/upload'">Ajout de documents</a>
</div>
</div>
</body>

View file

@ -8,6 +8,7 @@ import (
"sort"
"git.agecem.com/agecem/agecem-org/api"
"git.agecem.com/agecem/agecem-org/apirequest"
"git.agecem.com/agecem/agecem-org/apiresponse"
"git.agecem.com/agecem/agecem-org/models"
"git.agecem.com/agecem/agecem-org/webresponse"
@ -155,6 +156,7 @@ func (h *WebHandler) HandleAdminDocumentsUpload(c echo.Context) error {
}
func (h *WebHandler) HandleAdminDocumentsUploadPOST(c echo.Context) error {
var request apirequest.V1DocumentsPOST
var response webresponse.HandleAdminDocumentsUploadResponse
v1BucketsGET, err := h.ApiClient.ListBuckets()
@ -173,35 +175,79 @@ func (h *WebHandler) HandleAdminDocumentsUploadPOST(c echo.Context) error {
})
}
bucket := c.FormValue("bucket")
request.Params.Bucket = c.FormValue("bucket")
document, err := c.FormFile("document")
form, err := c.MultipartForm()
if err != nil {
response.StatusCode = http.StatusBadRequest
response.Message = "Formulaire invalide"
response.Message = "Impossible de téléverser"
response.Error = err.Error()
return c.Render(response.StatusCode, "admin-upload-html", response)
}
if form == nil {
response.StatusCode = http.StatusInternalServerError
response.Message = "Formulaire pointe vers une addresse mémoire nulle"
response.Error = "Formulaire pointe vers une addresse mémoire nulle"
uploadDocumentResponse, err := h.ApiClient.UploadDocument(bucket, document)
return c.Render(response.StatusCode, "admin-upload-html", response)
}
if len(form.File) == 0 {
response.StatusCode = http.StatusBadRequest
response.Message = "Veuillez sélectionner au moins 1 fichier à téléverser"
response.Error = "Input 'documents' ne contient aucun fichier"
return c.Render(response.StatusCode, "admin-upload-html", response)
}
for inputName, inputFileHeaders := range form.File {
if inputName == "documents" {
request.Data.Documents = inputFileHeaders
break
}
}
if request.Data.Documents == nil {
response.StatusCode = http.StatusBadRequest
response.Message = "Impossible d'obtenir les documents depuis le formulaire"
response.Error = "Impossible d'obtenir les documents depuis le formulaire"
return c.Render(response.StatusCode, "admin-upload-html", response)
}
uploadDocumentsResponse, err := h.ApiClient.UploadDocuments(request.Params.Bucket, request.Data.Documents...)
if err != nil {
response.StatusCode = uploadDocumentResponse.StatusCode
response.Message = uploadDocumentResponse.Message
response.Error = err.Error()
//TODO figure out pourquoi `err` n'est jamais `nil`
response.StatusCode = uploadDocumentsResponse.StatusCode
response.Message = uploadDocumentsResponse.Message
response.Error = fmt.Sprintf("%s. Détails: %s", err.Error(), uploadDocumentsResponse.Error)
/*
response.StatusCode = http.StatusInternalServerError
response.Message = fmt.Sprintf("api.(*API).UploadDocuments: %s", err)
response.Error = err.Error()
*/
return c.Render(response.StatusCode, "admin-upload-html", response)
}
//TODO figure out pourquoi on se rend jamais ici
// Format response
var info, status string
info = fmt.Sprintf("[%d] /public/documentation/%s/%s", uploadDocumentResponse.Data.Size, uploadDocumentResponse.Data.Bucket, uploadDocumentResponse.Data.Key)
for i, document := range uploadDocumentsResponse.Data.Documents {
info = fmt.Sprintf("%s[%d] /public/documentation/%s/%s (%dk) ok\n",
info, i, uploadDocumentsResponse.Data.Bucket, document.Key, document.Size)
}
status = uploadDocumentResponse.Message
status = uploadDocumentsResponse.Message
if errMsg := uploadDocumentsResponse.Error; errMsg != "" {
status = fmt.Sprintf("%s. Erreur: %s", status, errMsg)
}
response.StatusCode = http.StatusOK
response.Message = fmt.Sprintf("%s - %s", status, info)
response.Message = fmt.Sprintf("%s \n %s", status, info)
return c.Render(response.StatusCode, "admin-upload-html", response)
}